Error getting async data from Solis Pro (Ginlong) plataform with payload-CodePudding

I'm developing a web scraper to mine data from the Solis Pro platform (Ginlong), but I'm having problems getting the asynchronous data from the plants registered by the user. I'm using Selenium bs4 and the following has happened. The url is https://m.ginlong.com/pro/epc/plantview/view/doAsyncPlantList.json. I send a payload and in theory I should receive the data, but I am either receiving an error or only part of the data (only {status: 1}).

Plataform and payload

url = 'https://m.ginlong.com/'

chrome_options = Options()
chrome_options.add_argument("--headless")  
webdriver = Chrome(ChromeDriverManager().install(), options=chrome_options)
    
webdriver.get(url 'login.html')
webdriver.find_element(By.XPATH, '//button[@ng-click="typeChange()"]').click()
webdriver.find_element_by_name("username").send_keys(username)
webdriver.find_element_by_name("password").send_keys(password)

#webdriver.find_element(By.XPATH, '//button[@]').click()
webdriver.find_element(By.XPATH, '//button[@id="login"]').click()
soup = BeautifulSoup(webdriver.page_source, 'lxml')

headers = {
    "content-type": "application/x-www-form-urlencoded",
    "accept": "application/json",
    "user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Mobile Safari/537.36"
}

postData = {
   "plantArray": [{"plantId":1033610,"timezoneId":24,"updateTime":"2022-01-18"},{"plantId":1028383,"timezoneId":24,"updateTime":"2022-01-18"}]
}

response = webdriver.request('POST', url 'pro/epc/plantview/view/doAsyncPlantList.json', headers=headers, data=postData)

if response.status_code != 200:
    raise RuntimeError("A requisição falhou: %s", response)

data = BeautifulSoup(response.text, 'html.parser')
print(data.text)

The error that I'm getting

Exception: Unrecognized token 'plantId': was expecting ('true', 'false' or 'null')
 at [Source: plantId,timezoneId,updateTime,plantId,timezoneId,updateTime; line: 1, column: 8]

    com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'plantId': was expecting ('true', 'false' or 'null')
 at [Source: plantId,timezoneId,updateTime,plantId,timezoneId,updateTime; line: 1, column: 8]
    at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1702)
    at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:558)
    at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2836)
    at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:1899)
    at com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:749)
    at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:3847)
    at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:3792)
    at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2835)
    at com.igen.solarman.controller.epc.PlantViewController.doAsyncPlantList(PlantViewController.java:345)
    at com.igen.solarman.controller.epc.PlantViewController$$FastClassBySpringCGLIB$$70060edf.invoke()
    at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:720)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
    at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
    at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:655)
    at com.igen.solarman.controller.epc.PlantViewController$$EnhancerBySpringCGLIB$$b39c8e9a.doAsyncPlantList()
    at sun.reflect.GeneratedMethodAccessor1121.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:221)
    at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:136)
    at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:114)
    at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827)
    at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738)
    at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
    at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:963)
    at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
    at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
    at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:650)
    at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
    at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
    at com.alibaba.druid.support.http.WebStatFilter.doFilter(WebStatFilter.java:123)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
    at org.tuckey.web.filters.urlrewrite.RuleChain.handleRewrite(RuleChain.java:176)
    at org.tuckey.web.filters.urlrewrite.RuleChain.doRules(RuleChain.java:145)
    at org.tuckey.web.filters.urlrewrite.UrlRewriter.processRequest(UrlRewriter.java:92)
    at org.tuckey.web.filters.urlrewrite.UrlRewriteFilter.doFilter(UrlRewriteFilter.java:394)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
    at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:61)
    at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108)
    at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137)
    at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
    at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66)
    at org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449)
    at org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365)
    at org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90)
    at org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83)
    at org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383)
    at org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362)
    at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
    at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
    at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
    at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
    at org.apache.logging.log4j.web.Log4jServletFilter.doFilter(Log4jServletFilter.java:71)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:219)
    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:110)
    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:494)
    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:104)
    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:445)
    at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1137)
    at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
    at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
    at java.lang.Thread.run(Thread.java:745)

Data

Has anyone tried to get this data or know how I can resolve this error?

CodePudding user response：

Change: response = webdriver.request('POST', url 'pro/epc/plantview/view/doAsyncPlantList.json', headers=headers, data=postData)

to this: response = webdriver.request('POST', url 'pro/epc/plantview/view/doAsyncPlantList.json', headers=headers, json=json.dumps(postData))

(remember to import json) :)

I am not sure why this works but it does, for further reading see this discussion about the difference between data= and json= : Difference between data and json parameters in python requests package

Also, I've managed to get it work with requests only which should speed things up, note that I've had to change my url at the end to "cpro" not "pro" like yours since I don't have a pro account: "https://m.ginlong.com/pro/epc/plantview/view/doAsyncPlantList.json"

import requests
import json

s = requests.Session()

headers =   {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'}
url = 'https://m.ginlong.com/login.html'

resp = s.get(url,headers=headers)
print(resp)

login_url = 'https://m.ginlong.com/cpro/login/validateLogin.json'
payload = {
    'userName':'username',
    'userNameDisplay':'username',
    'password':'password',
    'lan':'2',
    'domain':'m.ginlong.com',
    'userType':'C'
    }

headers = {
    'accept':'application/json, text/plain, */*',
    'accept-encoding':'gzip, deflate, br',
    'content-type':'application/x-www-form-urlencoded',
    'origin':'https://m.ginlong.com',
    'referer':'https://m.ginlong.com/login.html',
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
    }

login = s.post(login_url,headers=headers,data=payload)
print(login)

new_plant_payload = {
    'name':'new_plant_2',
    'nmi':'',
    'type':'2',
    'gridType':'1',
    'searchInput':'',
    'positionAddressInput':'ChinaJiang Su ShengWu Xi ShiXin Wu QuTian An Zhi Hui Cheng A2',
    'rectangle':'120.36118,31.5065;',
    'lon':'120.36118',
    'lat':'31.5065',
    'address':'ChinaJiang Su ShengWu Xi ShiXin Wu QuTian An Zhi Hui Cheng A2',
    'country':'Jiang Su Sheng',
    'state':'Wu Xi Shi',
    'city':'Xin Wu Qu',
    'district':'Tian An Zhi Hui Cheng A2',
    'countryId':'44',
    'stateId':'',
    'cityId':'',
    'districtId':'',
    'power':'5.00',
    'timezoneId':'34',
    'direction':'180°',
    'angle':'26°',
    'currencyId':'19',
    'benchmarkPrice':'1.372',
    'price':'0.75',
    'priceNet':'1',
    'subsidy':'0.372',
    'subsidyYears':'20',
    'subsidyLocal':'0',
    'subsidyLocalYears':'0',
    'subsidyCity':'0',
    'subsidyCityYears':'0',
    'subsidyCounty':'0',
    'subsidyCountyYears':'0',
    'cost':'37500.00',
    'subsidyBuild':'1000',
    'percent':'0%',
    'interest':'0',
    'years':'',
    'repay':'1',
    'phone':''
    }

create_plant_headers = {
    'accept':'application/json',
    'content-type':'application/x-www-form-urlencoded',
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
    }

create_url = 'https://m.ginlong.com/cpro/epc/plantCreate/create/doSubmitPlant.json'

create = s.post(create_url,headers=create_plant_headers,data=new_plant_payload)
print(create)
print(create.text)

plant_id = create.json()['result']

async_url = 'https://m.ginlong.com/cpro/epc/plantview/view/doAsyncPlantList.json'
postData = {
   "plantArray": [{"plantId":plant_id,"timezoneId":24,"updateTime":"2022-01-18"}]
}
post_async = s.post(async_url,json=json.dumps(postData))

print(post_async)
print(post_async.text)