Skip to content

L
lcc-shape
Commit 4e366aaf authored by wang's avatar wang
Browse files
Options

Merge remote-tracking branch 'origin/tk' into wn11 

# Conflicts:
#	test_cookie.py
#	wn/src/src2.js
#	wn/src_run1.js
parents f7b96a30 b7b90cfe
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 18220 additions and 12294 deletions
package-lock.json
View file @ 4e366aaf
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
"domexception": "^2.0.1", "domexception": "^2.0.1",
"escodegen": "^2.0.0", "escodegen": "^2.0.0",
"express": "^4.18.2", "express": "^4.18.2",
"ffi-napi": "^4.0.3",
"fingerprint-generator": "^2.1.37", "fingerprint-generator": "^2.1.37",
"form-data": "^4.0.0", "form-data": "^4.0.0",
"ghost-cursor": "^1.1.18", "ghost-cursor": "^1.1.18",
...@@ -863,6 +864,44 @@ ...@@ -863,6 +864,44 @@
"node": ">= 0.10.0" "node": ">= 0.10.0"
} }
}, },
"node_modules/ffi-napi": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-4.0.3.tgz",
"integrity": "sha512-PMdLCIvDY9mS32RxZ0XGb95sonPRal8aqRhLbeEtWKZTe2A87qRFG9HjOhvG8EX2UmQw5XNRMIOT+1MYlWmdeg==",
"hasInstallScript": true,
"dependencies": {
"debug": "^4.1.1",
"get-uv-event-loop-napi-h": "^1.0.5",
"node-addon-api": "^3.0.0",
"node-gyp-build": "^4.2.1",
"ref-napi": "^2.0.1 || ^3.0.2",
"ref-struct-di": "^1.1.0"
},
"engines": {
"node": ">=10"
}
},
"node_modules/ffi-napi/node_modules/debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
"dependencies": {
"ms": "2.1.2"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/ffi-napi/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/finalhandler": { "node_modules/finalhandler": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmmirror.com/finalhandler/-/finalhandler-1.2.0.tgz", "resolved": "https://registry.npmmirror.com/finalhandler/-/finalhandler-1.2.0.tgz",
...@@ -1006,6 +1045,19 @@ ...@@ -1006,6 +1045,19 @@
"node": ">=4" "node": ">=4"
} }
}, },
"node_modules/get-symbol-from-current-process-h": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz",
"integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw=="
},
"node_modules/get-uv-event-loop-napi-h": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz",
"integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==",
"dependencies": {
"get-symbol-from-current-process-h": "^1.0.1"
}
},
"node_modules/ghost-cursor": { "node_modules/ghost-cursor": {
"version": "1.1.18", "version": "1.1.18",
"resolved": "https://registry.npmjs.org/ghost-cursor/-/ghost-cursor-1.1.18.tgz", "resolved": "https://registry.npmjs.org/ghost-cursor/-/ghost-cursor-1.1.18.tgz",
...@@ -1486,6 +1538,11 @@ ...@@ -1486,6 +1538,11 @@
"node": ">=10" "node": ">=10"
} }
}, },
"node_modules/node-addon-api": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz",
"integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A=="
},
"node_modules/node-fetch": { "node_modules/node-fetch": {
"version": "2.7.0", "version": "2.7.0",
"resolved": "https://registry.npmmirror.com/node-fetch/-/node-fetch-2.7.0.tgz", "resolved": "https://registry.npmmirror.com/node-fetch/-/node-fetch-2.7.0.tgz",
...@@ -1524,6 +1581,16 @@ ...@@ -1524,6 +1581,16 @@
"webidl-conversions": "^3.0.0" "webidl-conversions": "^3.0.0"
} }
}, },
"node_modules/node-gyp-build": {
"version": "4.8.0",
"resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.0.tgz",
"integrity": "sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og==",
"bin": {
"node-gyp-build": "bin.js",
"node-gyp-build-optional": "optional.js",
"node-gyp-build-test": "build-test.js"
}
},
"node_modules/node-releases": { "node_modules/node-releases": {
"version": "2.0.13", "version": "2.0.13",
"resolved": "https://registry.npmmirror.com/node-releases/-/node-releases-2.0.13.tgz", "resolved": "https://registry.npmmirror.com/node-releases/-/node-releases-2.0.13.tgz",
...@@ -1828,6 +1895,63 @@ ...@@ -1828,6 +1895,63 @@
"node": ">= 6" "node": ">= 6"
} }
}, },
"node_modules/ref-napi": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.3.tgz",
"integrity": "sha512-LiMq/XDGcgodTYOMppikEtJelWsKQERbLQsYm0IOOnzhwE9xYZC7x8txNnFC9wJNOkPferQI4vD4ZkC0mDyrOA==",
"hasInstallScript": true,
"dependencies": {
"debug": "^4.1.1",
"get-symbol-from-current-process-h": "^1.0.2",
"node-addon-api": "^3.0.0",
"node-gyp-build": "^4.2.1"
},
"engines": {
"node": ">= 10.0"
}
},
"node_modules/ref-napi/node_modules/debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
"dependencies": {
"ms": "2.1.2"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/ref-napi/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/ref-struct-di": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz",
"integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==",
"dependencies": {
"debug": "^3.1.0"
}
},
"node_modules/ref-struct-di/node_modules/debug": {
"version": "3.2.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
"dependencies": {
"ms": "^2.1.1"
}
},
"node_modules/ref-struct-di/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/requires-port": { "node_modules/requires-port": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
......
src_out.js
View file @ 4e366aaf
This diff is collapsed.
test_cookie.py
View file @ 4e366aaf
This diff is collapsed.
wn/Dockerfile
View file @ 4e366aaf
...@@ -10,11 +10,14 @@ RUN apt install -y make g++ ...@@ -10,11 +10,14 @@ RUN apt install -y make g++
COPY src/package.json /home/opt/lcc-wn/package.json COPY src/package.json /home/opt/lcc-wn/package.json
COPY src/package-lock.json /home/opt/lcc-wn/package-lock.json COPY src/package-lock.json /home/opt/lcc-wn/package-lock.json
RUN cd /home/opt/lcc-wn/ && npm i RUN cd /home/opt/lcc-wn/ && npm i
RUN npm i ffi-napi
COPY src/src.js /home/opt/lcc-wn/src.js COPY src/src.js /home/opt/lcc-wn/src.js
COPY src/src2.js /home/opt/lcc-wn/src2.js COPY src/src2.js /home/opt/lcc-wn/src2.js
COPY src/jsdom /home/opt/lcc-wn/jsdom COPY src/jsdom /home/opt/lcc-wn/jsdom
COPY src/js_dom_run.js /home/opt/lcc-wn/js_dom_run.js COPY src/js_dom_run.js /home/opt/lcc-wn/js_dom_run.js
RUN npm i sync-request COPY src/httpUtil.js /home/opt/lcc-wn/httpUtil.js
COPY src/tls-client-linux-ubuntu-amd64-v1.7.2.so /home/opt/lcc-wn/tls-client-linux-ubuntu-amd64-v1.7.2.so
COPY src/main.py /home/opt/lcc-wn/main.py COPY src/main.py /home/opt/lcc-wn/main.py
......
wn/httpUtil.js 0 → 100644
View file @ 4e366aaf
const ffi = require('ffi-napi');
// 判断设备是否为linux
const os = require('os');
if (os.platform() === 'linux') {
path = './tls-client-linux-ubuntu-amd64-v1.7.2.so';
} else {
path = './tls-client-arm64.dylib'
}
// load the tls-client shared package for your OS you are currently running your nodejs script (i'm running on mac)
const tlsClientLibrary = ffi.Library(path, {
'request': ['string', ['string']],
'getCookiesFromSession': ['string', ['string']],
'addCookiesToSession': ['string', ['string']],
'freeMemory': ["void", ['string']],
'destroyAll': ['string', []],
'destroySession': ['string', ['string']]
});
function requestSync(method, url, headers, proxy, data) {
const requestPayload = {
"tlsClientIdentifier": "chrome_117",
"followRedirects": true,
"insecureSkipVerify": false,
"withoutCookieJar": false,
"withDefaultCookieJar": false,
"isByteRequest": false,
"catchPanics": false,
"withDebug": false,
"forceHttp1": false,
"withRandomTLSExtensionOrder": false,
"timeoutSeconds": 5,
"timeoutMilliseconds": 0,
"sessionId": "my-session-id",
"proxyUrl": "",
"isRotatingProxy": false,
"certificatePinningHosts": {},
"headers": headers,
"headerOrder": [
"accept",
"user-agent",
"accept-encoding",
"accept-language"
],
"requestUrl": url,
"requestMethod": method,
"requestBody": data,
"requestCookies": []
}
// call the library with the requestPayload as string
const response = tlsClientLibrary.request(JSON.stringify(requestPayload));
// convert response string to json
const responseObject = JSON.parse(response)
console.log(responseObject)
tlsClientLibrary.freeMemory(responseObject.id)
const payload = {
sessionId: 'my-session-id',
url: "https://microsoft.com",
}
const cookiesResponse = tlsClientLibrary.getCookiesFromSession(JSON.stringify(payload))
const cookiesInSession = JSON.parse(cookiesResponse)
const destroySessionPayload = {
sessionId: 'my-session-id',
}
const destroySessionResponse = tlsClientLibrary.destroySession(JSON.stringify(destroySessionPayload))
const destroySessionResponseParsed = JSON.parse(destroySessionResponse)
// console.log(destroySessionResponseParsed)
return responseObject
}
module.exports = requestSync
\ No newline at end of file
wn/src/env.json
View file @ 4e366aaf
This diff is collapsed.
wn/src/httpUtil.js 0 → 100644
View file @ 4e366aaf
const ffi = require('ffi-napi');
// 判断设备是否为linux
const os = require('os');
if (os.platform() === 'linux') {
path = './tls-client-linux-ubuntu-amd64-v1.7.2.so';
} else {
path = './tls-client-arm64.dylib'
}
// load the tls-client shared package for your OS you are currently running your nodejs script (i'm running on mac)
const tlsClientLibrary = ffi.Library(path, {
'request': ['string', ['string']],
'getCookiesFromSession': ['string', ['string']],
'addCookiesToSession': ['string', ['string']],
'freeMemory': ["void", ['string']],
'destroyAll': ['string', []],
'destroySession': ['string', ['string']]
});
function requestSync(method, url, headers, proxy, data) {
const requestPayload = {
"tlsClientIdentifier": "chrome_117",
"followRedirects": true,
"insecureSkipVerify": false,
"withoutCookieJar": false,
"withDefaultCookieJar": false,
"isByteRequest": false,
"catchPanics": false,
"withDebug": false,
"forceHttp1": false,
"withRandomTLSExtensionOrder": false,
"timeoutSeconds": 5,
"timeoutMilliseconds": 0,
"sessionId": "my-session-id",
"proxyUrl": "",
"isRotatingProxy": false,
"certificatePinningHosts": {},
"headers": headers,
"headerOrder": [
"accept",
"user-agent",
"accept-encoding",
"accept-language"
],
"requestUrl": url,
"requestMethod": method,
"requestBody": data,
"requestCookies": []
}
// call the library with the requestPayload as string
const response = tlsClientLibrary.request(JSON.stringify(requestPayload));
// convert response string to json
const responseObject = JSON.parse(response)
console.log(responseObject)
tlsClientLibrary.freeMemory(responseObject.id)
const payload = {
sessionId: 'my-session-id',
url: "https://microsoft.com",
}
const cookiesResponse = tlsClientLibrary.getCookiesFromSession(JSON.stringify(payload))
const cookiesInSession = JSON.parse(cookiesResponse)
const destroySessionPayload = {
sessionId: 'my-session-id',
}
const destroySessionResponse = tlsClientLibrary.destroySession(JSON.stringify(destroySessionPayload))
const destroySessionResponseParsed = JSON.parse(destroySessionResponse)
// console.log(destroySessionResponseParsed)
return responseObject
}
module.exports = requestSync
\ No newline at end of file
wn/src/js_dom_run.js
View file @ 4e366aaf
This source diff could not be displayed because it is too large. You can view the blob instead.
wn/src/js_dom_run1.js
View file @ 4e366aaf
This diff is collapsed.
wn/src/main.py
View file @ 4e366aaf
...@@ -6,14 +6,13 @@ import subprocess ...@@ -6,14 +6,13 @@ import subprocess
import time import time
import os import os
import loguru import loguru
# import requests import requests
import json import json
# import tls_client # import tls_client
from bson import ObjectId from bson import ObjectId
import hashlib import hashlib
from pymongo import MongoClient from pymongo import MongoClient
from loguru import logger from loguru import logger
from curl_cffi import requests
def get_random_location(): def get_random_location():
return '' return ''
...@@ -221,32 +220,31 @@ while True: ...@@ -221,32 +220,31 @@ while True:
# proxy = f'http://ynfflcc2023.hotmail.com:gdcfh9@69.30.227.194:{random.randint(20001, 20250)}' # proxy = f'http://ynfflcc2023.hotmail.com:gdcfh9@69.30.227.194:{random.randint(20001, 20250)}'
proxy2 = f'http://user-uni003-region-us-sessid-{random.randint(1000,5000)}-sesstime-5-keep-true:q39CEBTs5A5YQXor@pr.roxlabs.cn:4600' proxy2 = f'http://user-uni003-region-us-sessid-{random.randint(1000,5000)}-sesstime-5-keep-true:q39CEBTs5A5YQXor@pr.roxlabs.cn:4600'
proxy = f'http://user-uni003-region-us-sessid-{random.randint(1000,5000)}-sesstime-5-keep-true:q39CEBTs5A5YQXor@pr.roxlabs.cn:4600' # proxy = f'http://user-uni003-region-us-sessid-{random.randint(1000,5000)}-sesstime-5-keep-true:q39CEBTs5A5YQXor@pr.roxlabs.cn:4600'
print(proxy)
# proxy = f'http://unfflcc:76cc14-47b8dd-1f8ace-827836-0c740e@198.147.25.26:{int(random.randint(10000, 11149))}' # proxy = f'http://unfflcc:76cc14-47b8dd-1f8ace-827836-0c740e@198.147.25.26:{int(random.randint(10000, 11149))}'
# proxy = 'http://user-unifflcc-region-us:q39CEBTs5A5YQXor@pr.roxlabs.cn:4600' # proxy = 'http://user-unifflcc-region-us:q39CEBTs5A5YQXor@pr.roxlabs.cn:4600'
tlsV = f'chrome_{random.randint(105, 113)}' tlsV = f'chrome_{random.randint(105, 113)}'
# proxy = random.choice([proxy1, proxy2]) # proxy = random.choice([proxy1, proxy2])
# proxy = get_proxy()['url'] proxy = get_proxy()['url']
logger.info(f"success:{success} proxy: {proxy} ") logger.info(f"success:{success} proxy: {proxy} ")
tlsV = 'chrome_112' tlsV = 'chrome_112'
session = requests.Session(impersonate='chrome110') # session = requests.Session(impersonate='chrome110')
# session = tls_client.Session(tlsV) # # session = tls_client.Session(tlsV)
# session.timeout_seconds = 15 # # session.timeout_seconds = 15
# session = requests.session() # # session = requests.session()
session.proxies = { # session.proxies = {
'http': proxy, # 'http': proxy,
'https': proxy, # 'https': proxy,
} # }
# session.verify = False # # session.verify = False
if platform.uname().system == 'Darwin': # if platform.uname().system == 'Darwin':
session.proxies = { # session.proxies = {
'http': 'http://127.0.0.1:7890', # 'http': 'http://127.0.0.1:7890',
'https': 'http://127.0.0.1:7890', # 'https': 'http://127.0.0.1:7890',
} # }
selectId = random.choice(list(envs)) selectId = random.choice(list(envs))
# selectId = 'a74048568ce5f6140df6d956' # selectId = '197d94c3099d110aef4e0a52'
if usedTime > 5: if usedTime > 5:
# c = session.get('https://www.southwest.com/assets/app/scripts/swa-common.js').text # c = session.get('https://www.southwest.com/assets/app/scripts/swa-common.js').text
response = requests.post('http://52.52.23.116/tls/forward', response = requests.post('http://52.52.23.116/tls/forward',
...@@ -304,17 +302,18 @@ setTimeout(r=> { ...@@ -304,17 +302,18 @@ setTimeout(r=> {
# f.write(code + '\n\n' + get_random_location() + ';;' + c) # f.write(code + '\n\n' + get_random_location() + ';;' + c)
code1 = open('js_dom_run.js','r').read() code1 = open('js_dom_run.js','r').read()
# with open('js_dom_run1.js', 'w', encoding='utf-8') as f: with open('js_dom_run1.js', 'w', encoding='utf-8') as f:
# f.write(code1 + '\n\n' + c) f.write(code1 + '\n\n' + c)
t1 = time.time() t1 = time.time()
# token = list(db.token_shape.aggregate( [ {'$match': {'update_time':{'$gte': int(time.time() * 1000) - 60*60*12}}}, { '$sample': { 'size': 1 } } ] ))[0]['_id'] # token = list(db.token_shape.aggregate( [ {'$match': {'update_time':{'$gte': int(time.time() * 1000) - 60*60*12}}}, { '$sample': { 'size': 1 } } ] ))[0]['_id']
# token = '11' # token = '11'
# r = requests.get('http://127.0.0.1:5000/getinfo').json()['message'] # r = requests.get('http://127.0.0.1:5000/getinfo').json()['message']
# r = requests.get('http://127.0.0.1:3001').json() # r = requests.get('http://127.0.0.1:3001').json()
logger.info("start execute js")
# p = subprocess.Popen(['node', 'js_dom_run1.js', selectId, '11'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, # p = subprocess.Popen(['node', 'js_dom_run1.js', selectId, '11'], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
# p = subprocess.Popen(['node', 'src_run_chrome_105.js', selectId, '11', token], stdin=subprocess.PIPE, stdout=subprocess.PIPE, # p = subprocess.Popen(['node', 'src_run_chrome_105.js', selectId, '11', token], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
# p = subprocess.Popen(['node', 'src_run_company_chrome.js', selectId, '11', token], stdin=subprocess.PIPE, stdout=subprocess.PIPE, # p = subprocess.Popen(['node', 'src_run_company_chrome.js', selectId, '11', token], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
logger.info("start execute js")
p = subprocess.Popen(['node', 'src_run1.js', selectId, '11', token, proxy], stdin=subprocess.PIPE, stdout=subprocess.PIPE, p = subprocess.Popen(['node', 'src_run1.js', selectId, '11', token, proxy], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE)
output, err = p.communicate(timeout=10) output, err = p.communicate(timeout=10)
...@@ -330,8 +329,8 @@ setTimeout(r=> { ...@@ -330,8 +329,8 @@ setTimeout(r=> {
# ) # )
usedTime += 1 usedTime += 1
# print(r) # print(r)
xhr_url = r.pop('xhr_url') # xhr_url = r.pop('xhr_url')
xhr_res = r.pop('xhr_res') # xhr_res = r.pop('xhr_res')
# print('xhr_url', xhr_url) # print('xhr_url', xhr_url)
# print('xhr_res', xhr_res) # print('xhr_res', xhr_res)
headers.update(r) headers.update(r)
......
wn/src/src2.js
View file @ 4e366aaf
This diff is collapsed.
wn/src_run1.js
View file @ 4e366aaf
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment