I have the following Python script who's purpose is to copy the entire Contract Source Code from https://bscscan.com/address/0xf811e41dc5271814ceb9aaf4f148478d5c6a0134#code using the "Copy source code to clipboard" button, located below the "Outline" and "More Options" buttons.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_argument("--headless")
driver = webdriver.Chrome(options=options)
driver.get('https://bscscan.com/address/0xf811e41dc5271814ceb9aaf4f148478d5c6a0134#code')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[2]/div/div/div/button'))).click()
WebDriverWait(driver, 20).until(
EC.element_to_be_clickable(
(By.XPATH, '/html/body/div[1]/main/div[4]/div[3]/div[2]/div/div[7]/div[2]/div[2]/div[1]/div[2]/span/a[1]')
)
).click()
driver.quit()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[2]/div/div/div/button'))).click()
is used to click an "allow cookies" pop-up which blocks the button I want when the page opens.
When running in headless mode, which is enabled using options.add_argument("--headless") I don't get an error but the text is not copied. When I comment out options.add_argument("--headless") and run the code with the normal window, the text is copied.
I plan to run this script in the background to scrape contract information from multiple cryptocurrency tokens, so being able to run it in headless mode is important.
CodePudding user response：
I guess you should set the window size in headless mode, this will make your headless session to be similar to the regular session run.
Also, you should improve your locators.
Let me know if this works better:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_argument("--headless")
options.add_argument("window-size=1920,1080")
driver = webdriver.Chrome(options=options)
driver.get('https://bscscan.com/address/0xf811e41dc5271814ceb9aaf4f148478d5c6a0134#code')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[@id='btnCookie']"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.justify-content-end a.js-clipboard "))).click()
driver.quit()
CodePudding user response：
To print the desired text you need to induce WebDriverWait for the visibility_of_element_located() and you can use either of the following Locator Strategies:
Using CSS_SELECTOR and text attribute:
from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.service import Service from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC options = Options() options.headless = True options.add_argument("start-maximized") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('excludeSwitches', ['enable-logging']) options.add_experimental_option('useAutomationExtension', False) options.add_argument('--disable-blink-features=AutomationControlled') s = Service('C:\\BrowserDrivers\\chromedriver.exe') driver = webdriver.Chrome(service=s, options=options) driver.get("https://bscscan.com/address/0xf811e41dc5271814ceb9aaf4f148478d5c6a0134#code") WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button#btnCookie"))).click() print(WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.ace_layer.ace_text-layer"))).text)Using XPATH and text attribute:
from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.service import Service from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC options = Options() options.headless = True options.add_argument("start-maximized") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('excludeSwitches', ['enable-logging']) options.add_experimental_option('useAutomationExtension', False) options.add_argument('--disable-blink-features=AutomationControlled') s = Service('C:\\BrowserDrivers\\chromedriver.exe') driver = webdriver.Chrome(service=s, options=options) driver.get("https://bscscan.com/address/0xf811e41dc5271814ceb9aaf4f148478d5c6a0134#code") WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[@id='btnCookie']"))).click() print(WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//div[@class='ace_layer ace_text-layer']"))).text)Note : You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC
You can find a relevant discussion in How to retrieve the text of a WebElement using Selenium - Python
Alternative
To click on the WebElement with data-original-title set as Copy source code to clipboard and to print to the console you can use the clipboard module as follows:
Code Block:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import clipboard
options = Options()
options.headless = True
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('excludeSwitches', ['enable-logging'])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument('--disable-blink-features=AutomationControlled')
s = Service('C:\\BrowserDrivers\\chromedriver.exe')
driver = webdriver.Chrome(service=s, options=options)
driver.get("https://bscscan.com/address/0xf811e41dc5271814ceb9aaf4f148478d5c6a0134#code")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button#btnCookie"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a[data-original-title='Copy source code to clipboard']"))).click()
print(clipboard.paste())
Console Output:
/**
*Submitted for verification at BscScan.com on 2021-11-21
*/
// SPDX-License-Identifier: UNLICENSED
pragma solidity ^0.7.4;
library SafeMath {
function add(uint256 a, uint256 b) internal pure returns (uint256) {
uint256 c = a b;
require(c >= a, "SafeMath: addition overflow");
return c;
}
function sub(uint256 a, uint256 b) internal pure returns (uint256) {
return sub(a, b, "SafeMath: subtraction overflow");
}
function sub(uint256 a, uint256 b, string memory errorMessage) internal pure returns (uint256) {
require(b <= a, errorMessage);
uint256 c = a - b;
return c;
}
function mul(uint256 a, uint256 b) internal pure returns (uint256) {
if (a == 0) {
return 0;
}
uint256 c = a * b;
require(c / a == b, "SafeMath: multiplication overflow");
return c;
}
function div(uint256 a, uint256 b) internal pure returns (uint256) {
return div(a, b, "SafeMath: division by zero");
}
function div(uint256 a, uint256 b, string memory errorMessage) internal pure returns (uint256) {
require(b > 0, errorMessage);
uint256 c = a / b;
return c;
}
}
/**
* BEP20 standard interface.
*/
interface IBEP20 {
function totalSupply() external view returns (uint256);
function decimals() external view returns (uint8);
function symbol() external view returns (string memory);
function name() external view returns (string memory);
function getOwner() external view returns (address);
function balanceOf(address account) external view returns (uint256);
function transfer(address recipient, uint256 amount) external returns (bool);
function allowance(address _owner, address spender) external view returns (uint256);
function approve(address spender, uint256 amount) external returns (bool);
function transferFrom(address sender, address recipient, uint256 amount) external returns (bool);
event Transfer(address indexed from, address indexed to, uint256 value);
event Approval(address indexed owner, address indexed spender, uint256 value);
}
abstract contract Auth {
address internal owner;
mapping (address => bool) internal authorizations;
constructor(address _owner) {
owner = _owner;
authorizations[_owner] = true;
}
modifier onlyOwner() {
require(isOwner(msg.sender), "!OWNER"); _;
}
modifier authorized() {
require(isAuthorized(msg.sender), "!AUTHORIZED"); _;
}
function authorize(address adr) public onlyOwner {
authorizations[adr] = true;
}
function unauthorize(address adr) public onlyOwner {
authorizations[adr] = false;
}
function isOwner(address account) public view returns (bool) {
return account == owner;
}
function isAuthorized(address adr) public view returns (bool) {
return authorizations[adr];
}
function transferOwnership(address payable adr) public onlyOwner {
owner = adr;
authorizations[adr] = true;
emit OwnershipTransferred(adr);
}
event OwnershipTransferred(address owner);
}
interface IDEXFactory {
function createPair(address tokenA, address tokenB) external returns (address pair);
}
interface IDEXRouter {
function factory() external pure returns (address);
function WETH() external pure returns (address);
function addLiquidity(
address tokenA,
address tokenB,
uint amountADesired,
uint amountBDesired,
uint amountAMin,
uint amountBMin,
address to,
uint deadline
) external returns (uint amountA, uint amountB, uint liquidity);
function addLiquidityETH(
address token,
uint amountTokenDesired,
uint amountTokenMin,
uint amountETHMin,
address to,
uint deadline
) external payable returns (uint amountToken, uint amountETH, uint liquidity);
function swapExactTokensForTokensSupportingFeeOnTransferTokens(
uint amountIn,
uint amountOutMin,
address[] calldata path,
address to,
uint deadline
) external;
function swapExactETHForTokensSupportingFeeOnTransferTokens(
uint amountOutMin,
address[] calldata path,
address to,
uint deadline
) external payable;
function swapExactTokensForETHSupportingFeeOnTransferTokens(
uint amountIn,
uint amountOutMin,
address[] calldata path,
address to,
uint deadline
) external;
}
interface IDividendDistributor {
function setDistributionCriteria(uint256 _minPeriod, uint256 _minDistribution) external;
function setShare(address shareholder, uint256 amount) external;
function deposit() external payable;
function process(uint256 gas) external;
}
contract DividendDistributor is IDividendDistributor {
using SafeMath for uint256;
address _token;
struct Share {
uint256 amount;
uint256 totalExcluded;
uint256 totalRealised;
}
IBEP20 RWRD = IBEP20(0xe9e7CEA3DedcA5984780Bafc599bD69ADd087D56);
address WBNB = 0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c;
IDEXRouter router;
address[] shareholders;
mapping (address => uint256) shareholderIndexes;
mapping (address => uint256) shareholderClaims;
mapping (address => Share) public shares;
uint256 public totalShares;
uint256 public totalDividends;
uint256 public totalDistributed;
uint256 public dividendsPerShare;
uint256 public dividendsPerShareAccuracyFactor = 10 ** 36;
uint256 public minPeriod = 45 * 60;
uint256 public minDistribution = 1 * (10 ** 19);
uint256 currentIndex;
bool initialized;
modifier initialization() {
require(!initialized);
_;
initialized = true;
}
modifier onlyToken() {
require(msg.sender == _token); _;
}
constructor (address _router) {
router = _router != address(0)
? IDEXRouter(_router)
: IDEXRouter(0x10ED43C718714eb63d5aA57B78B54704E256024E);
_token = msg.sender;
}
function setDistributionCriteria(uint256 _minPeriod, uint256 _minDistribution) external override onlyToken {
minPeriod = _minPeriod;
minDistribution = _minDistribution;
}
function setShare(address shareholder, uint256 amount) external override onlyToken {
if(shares[shareholder].amount > 0){
distributeDividend(shareholder);
}
if(amount > 0 && shares[shareholder].amount == 0){
addShareholder(shareholder);
}else if(amount == 0 && shares[shareholder].amount > 0){
removeShareholder(shareholder);
}
totalShares = totalShares.sub(shares[shareholder].amount).add(amount);
shares[shareholder].amount = amount;
shares[shareholder].totalExcluded = getCumulativeDividends(shares[shareholder].amount);
}
function deposit() external payable override onlyToken {
uint256 balanceBefore = RWRD.balanceOf(address(this));
address[] memory path = new address[](2);
path[0] = WBNB;
path[1] = address(RWRD);
router.swapExactETHForTokensSupportingFeeOnTransferTokens{value: msg.value}(
0,
path,
address(this),
block.timestamp
);
uint256 amount = RWRD.balanceOf(address(this)).sub(balanceBefore);
totalDividends = totalDividends.add(amount);
dividendsPerShare = dividendsPerShare.add(dividendsPerShareAccuracyFactor.mul(amount).div(totalShares));
}
function process(uint256 gas) external override onlyToken {
uint256 shareholderCount = shareholders.length;
if(shareholderCount == 0) { return; }
uint256 gasUsed = 0;
uint256 gasLeft = gasleft();
uint256 iterations = 0;
while(gasUsed < gas && iterations < shareholderCount) {
if(currentIndex >= shareholderCount){
currentIndex = 0;
}
if(shouldDistribute(shareholders[currentIndex])){
distributeDividend(shareholders[currentIndex]);
}
gasUsed = gasUsed.add(gasLeft.sub(gasleft()));
gasLeft = gasleft();
currentIndex ;
iterations ;
}
}
function shouldDistribute(address shareholder) internal view returns (bool) {
return shareholderClaims[shareholder] minPeriod < block.timestamp
&& getUnpaidEarnings(shareholder) > minDistribution;
}
function distributeDividend(address shareholder) internal {
if(shares[shareholder].amount == 0){ return; }
uint256 amount = getUnpaidEarnings(shareholder);
if(amount > 0){
totalDistributed = totalDistributed.add(amount);
RWRD.transfer(shareholder, amount);
shareholderClaims[shareholder] = block.timestamp;
shares[shareholder].totalRealised = shares[shareholder].totalRealised.add(amount);
shares[shareholder].totalExcluded = getCumulativeDividends(shares[shareholder].amount);
}
}
function claimDividend() external {
distributeDividend(msg.sender);
}
function getUnpaidEarnings(address shareholder) public view returns (uint256) {
if(shares[shareholder].amount == 0){ return 0; }
uint256 shareholderTotalDividends = getCumulativeDividends(shares[shareholder].amount);
uint256 shareholderTotalExcluded = shares[shareholder].totalExcluded;
if(shareholderTotalDividends <= shareholderTotalExcluded){ return 0; }
return shareholderTotalDividends.sub(shareholderTotalExcluded);
}
function getCumulativeDividends(uint256 share) internal view returns (uint256) {
return share.mul(dividendsPerShare).div(dividendsPerShareAccuracyFactor);
}
function addShareholder(address shareholder) internal {
shareholderIndexes[shareholder] = shareholders.length;
shareholders.push(shareholder);
}
function removeShareholder(address shareholder) internal {
shareholders[shareholderIndexes[shareholder]] = shareholders[shareholders.length-1];
shareholderIndexes[shareholders[shareholders.length-1]] = shareholderIndexes[shareholder];
shareholders.pop();
}
}
contract CASHSLOTHTOKEN is IBEP20, Auth {
using SafeMath for uint256;
address WBNB = 0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c;
address DEAD = 0x000000000000000000000000000000000000dEaD;
address ZERO = 0x0000000000000000000000000000000000000000;
string constant _name = "CashSlothToken";
string constant _symbol = "CSH";
uint8 constant _decimals = 2;
uint256 _totalSupply = 1 * 10**15 * 10**_decimals;
uint256 public _maxTxAmount = _totalSupply * 3 / 100;
uint256 public _maxWalletToken = _totalSupply * 5 / 100;
mapping (address => uint256) _balances;
mapping (address => mapping (address => uint256)) _allowances;
bool public blacklistMode = true;
mapping (address => bool) public isBlacklisted;
mapping (address => bool) isFeeExempt;
mapping (address => bool) isTxLimitExempt;
mapping (address => bool) isTimelockExempt;
mapping (address => bool) isDividendExempt;
uint256 public liquidityFee = 2;
uint256 public reflectionFee = 6;
uint256 public marketingFee = 5;
uint256 public teamFee = 1;
uint256 public burnFee = 1;
uint256 public totalFee = marketingFee reflectionFee liquidityFee teamFee burnFee;
uint256 public feeDenominator = 100;
uint256 public deadBlocks = 0;
uint256 public launchedAt = 0;
uint256 public sellMultiplier = 113;
address public autoLiquidityReceiver;
address public marketingFeeReceiver;
address public teamFeeReceiver;
address public burnFeeReceiver;
uint256 targetLiquidity = 99;
uint256 targetLiquidityDenominator = 100;
IDEXRouter public router;
address public pair;
bool public tradingOpen = false;
DividendDistributor public distributor;
uint256 distributorGas = 500000;
bool public buyCooldownEnabled = false;
uint8 public cooldownTimerInterval = 60;
mapping (address => uint) private cooldownTimer;
bool public swapEnabled = true;
uint256 public swapThreshold = _totalSupply * 8 / 10000;
bool inSwap;
modifier swapping() { inSwap = true; _; inSwap = false; }
constructor () Auth(msg.sender) {
router = IDEXRouter(0x10ED43C718714eb63d5aA57B78B54704E256024E);
pair = IDEXFactory(router.factory()).createPair(WBNB, address(this));
_allowances[address(this)][address(router)] = uint256(-1);
distributor = new DividendDistributor(address(router));
isFeeExempt[msg.sender] = true;
isTxLimitExempt[msg.sender] = true;
isTimelockExempt[msg.sender] = true;
isTimelockExempt[DEAD] = true;
isTimelockExempt[address(this)] = true;
isDividendExempt[pair] = true;
isDividendExempt[address(this)] = true;
isDividendExempt[DEAD] = true;
autoLiquidityReceiver = DEAD;
marketingFeeReceiver = 0xEE09d19caB47E66A957e70d5e186ff9448f0d987;
teamFeeReceiver = 0x277BdadF7A82Ab1a9C5Cac664abfdF748aFF3486;
burnFeeReceiver = DEAD;
_balances[msg.sender] = _totalSupply;
emit Transfer(address(0), msg.sender, _totalSupply);
}
receive() external payable { }
function totalSupply() external view override returns (uint256) { return _totalSupply; }
function decimals() external pure override returns (uint8) { return _decimals; }
function symbol() external pure override returns (string memory) { return _symbol; }
function name() external pure override returns (string memory) { return _name; }
function getOwner() external view override returns (address) { return owner; }
function balanceOf(address account) public view override returns (uint256) { return _balances[account]; }
function allowance(address holder, address spender) external view override returns (uint256) { return _allowances[holder][spender]; }
function approve(address spender, uint256 amount) public override returns (bool) {
_allowances[msg.sender][spender] = amount;
emit Approval(msg.sender, spender, amount);
return true;
}
function approveMax(address spender) external returns (bool) {
return approve(spender, uint256(-1));
}
function transfer(address recipient, uint256 amount) external override returns (bool) {
return _transferFrom(msg.sender, recipient, amount);
}
function transferFrom(address sender, address recipient, uint256 amount) external override returns (bool) {
if(_allowances[sender][msg.sender] != uint256(-1)){
_allowances[sender][msg.sender] = _allowances[sender][msg.sender].sub(amount, "Insufficient Allowance");
}
return _transferFrom(sender, recipient, amount);
}
function setMaxWalletPercent_base1000(uint256 maxWallPercent_base1000) external onlyOwner() {
_maxWalletToken = (_totalSupply * maxWallPercent_base1000 ) / 1000;
}
function setMaxTxPercent_base1000(uint256 maxTXPercentage_base1000) external onlyOwner() {
_maxTxAmount = (_totalSupply * maxTXPercentage_base1000 ) / 1000;
}
function setTxLimit(uint256 amount) external authorized {
_maxTxAmount = amount;
}
function _transferFrom(address sender, address recipient, uint256 amount) internal returns (bool) {
if(inSwap){ return _basicTransfer(sender, recipient, amount); }
if(!authorizations[sender] && !authorizations[recipient]){
require(tradingOpen,"Trading not open yet");
}
// Blacklist
if(blacklistMode){
require(!isBlacklisted[sender] && !isBlacklisted[recipient],"Blacklisted");
}
if (!authorizations[sender] && recipient != address(this) && recipient != address(DEAD) && recipient != pair && recipient != marketingFeeReceiver && recipient != teamFeeReceiver && recipient != autoLiquidityReceiver && recipient != burnFeeReceiver){
uint256 heldTokens = balanceOf(recipient);
require((heldTokens amount) <= _maxWalletToken,"Total Holding is currently limited, you can not buy that much.");}
if (sender == pair &&
buyCooldownEnabled &&
!isTimelockExempt[recipient]) {
require(cooldownTimer[recipient] < block.timestamp,"Please wait for 1min between two buys");
cooldownTimer[recipient] = block.timestamp cooldownTimerInterval;
}
// Checks max transaction limit
checkTxLimit(sender, amount);
if(shouldSwapBack()){ swapBack(); }
//Exchange tokens
_balances[sender] = _balances[sender].sub(amount, "Insufficient Balance");
uint256 amountReceived = (!shouldTakeFee(sender) || !shouldTakeFee(recipient)) ? amount : takeFee(sender, amount,(recipient == pair));
_balances[recipient] = _balances[recipient].add(amountReceived);
// Dividend tracker
if(!isDividendExempt[sender]) {
try distributor.setShare(sender, _balances[sender]) {} catch {}
}
if(!isDividendExempt[recipient]) {
try distributor.setShare(recipient, _balances[recipient]) {} catch {}
}
try distributor.process(distributorGas) {} catch {}
emit Transfer(sender, recipient, amountReceived);
return true;
}
function _basicTransfer(address sender, address recipient, uint256 amount) internal returns (bool) {
_balances[sender] = _balances[sender].sub(amount, "Insufficient Balance");
_balances[recipient] = _balances[recipient].add(amount);
emit Transfer(sender, recipient, amount);
return true;
}
function checkTxLimit(address sender, uint256 amount) internal view {
require(amount <= _maxTxAmount || isTxLimitExempt[sender], "TX Limit Exceeded");
}
...
...
// enable cooldown between trades
function cooldownEnabled(bool _status, uint8 _interval) public onlyOwner {
buyCooldownEnabled = _status;
cooldownTimerInterval = _interval;
}
function swapBack() internal swapping {
uint256 dynamicLiquidityFee = isOverLiquified(targetLiquidity, targetLiquidityDenominator) ? 0 : liquidityFee;
uint256 amountToLiquify = swapThreshold.mul(dynamicLiquidityFee).div(totalFee).div(2);
uint256 amountToSwap = swapThreshold.sub(amountToLiquify);
address[] memory path = new address[](2);
path[0] = address(this);
path[1] = WBNB;
uint256 balanceBefore = address(this).balance;
router.swapExactTokensForETHSupportingFeeOnTransferTokens(
amountToSwap,
0,
path,
address(this),
block.timestamp
);
uint256 amountBNB = address(this).balance.sub(balanceBefore);
uint256 totalBNBFee = totalFee.sub(dynamicLiquidityFee.div(2));
uint256 amountBNBLiquidity = amountBNB.mul(dynamicLiquidityFee).div(totalBNBFee).div(2);
uint256 amountBNBReflection = amountBNB.mul(reflectionFee).div(totalBNBFee);
uint256 amountBNBMarketing = amountBNB.mul(marketingFee).div(totalBNBFee);
uint256 amountBNBTeam = amountBNB.mul(teamFee).div(totalBNBFee);
try distributor.deposit{value: amountBNBReflection}() {} catch {}
(bool tmpSuccess,) = payable(marketingFeeReceiver).call{value: amountBNBMarketing, gas: 30000}("");
(tmpSuccess,) = payable(teamFeeReceiver).call{value: amountBNBTeam, gas: 30000}("");
// only to supress warning msg
tmpSuccess = false;
if(amountToLiquify > 0){
router.addLiquidityETH{value: amountBNBLiquidity}(
address(this),
amountToLiquify,
0,
0,
autoLiquidityReceiver,
block.timestamp
);
emit AutoLiquify(amountBNBLiquidity, amountToLiquify);
}
}
...
...
// ~by @monkey_shanti