Skip to content

L
lcc-shape
Commit 4e366aaf authored by wang's avatar wang
Browse files
Options

Merge remote-tracking branch 'origin/tk' into wn11 

# Conflicts:
#	test_cookie.py
#	wn/src/src2.js
#	wn/src_run1.js
parents f7b96a30 b7b90cfe
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 18220 additions and 12294 deletions
package-lock.json
View file @ 4e366aaf
......@@ -20,6 +20,7 @@
"domexception": "^2.0.1",
"escodegen": "^2.0.0",
"express": "^4.18.2",
"ffi-napi": "^4.0.3",
"fingerprint-generator": "^2.1.37",
"form-data": "^4.0.0",
"ghost-cursor": "^1.1.18",
......@@ -863,6 +864,44 @@
"node": ">= 0.10.0"
}
},
"node_modules/ffi-napi": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-4.0.3.tgz",
"integrity": "sha512-PMdLCIvDY9mS32RxZ0XGb95sonPRal8aqRhLbeEtWKZTe2A87qRFG9HjOhvG8EX2UmQw5XNRMIOT+1MYlWmdeg==",
"hasInstallScript": true,
"dependencies": {
"debug": "^4.1.1",
"get-uv-event-loop-napi-h": "^1.0.5",
"node-addon-api": "^3.0.0",
"node-gyp-build": "^4.2.1",
"ref-napi": "^2.0.1 || ^3.0.2",
"ref-struct-di": "^1.1.0"
},
"engines": {
"node": ">=10"
}
},
"node_modules/ffi-napi/node_modules/debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
"dependencies": {
"ms": "2.1.2"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/ffi-napi/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/finalhandler": {
"version": "1.2.0",
"resolved": "https://registry.npmmirror.com/finalhandler/-/finalhandler-1.2.0.tgz",
......@@ -1006,6 +1045,19 @@
"node": ">=4"
}
},
"node_modules/get-symbol-from-current-process-h": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz",
"integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw=="
},
"node_modules/get-uv-event-loop-napi-h": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz",
"integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==",
"dependencies": {
"get-symbol-from-current-process-h": "^1.0.1"
}
},
"node_modules/ghost-cursor": {
"version": "1.1.18",
"resolved": "https://registry.npmjs.org/ghost-cursor/-/ghost-cursor-1.1.18.tgz",
......@@ -1486,6 +1538,11 @@
"node": ">=10"
}
},
"node_modules/node-addon-api": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz",
"integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A=="
},
"node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmmirror.com/node-fetch/-/node-fetch-2.7.0.tgz",
......@@ -1524,6 +1581,16 @@
"webidl-conversions": "^3.0.0"
}
},
"node_modules/node-gyp-build": {
"version": "4.8.0",
"resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.0.tgz",
"integrity": "sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og==",
"bin": {
"node-gyp-build": "bin.js",
"node-gyp-build-optional": "optional.js",
"node-gyp-build-test": "build-test.js"
}
},
"node_modules/node-releases": {
"version": "2.0.13",
"resolved": "https://registry.npmmirror.com/node-releases/-/node-releases-2.0.13.tgz",
......@@ -1828,6 +1895,63 @@
"node": ">= 6"
}
},
"node_modules/ref-napi": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.3.tgz",
"integrity": "sha512-LiMq/XDGcgodTYOMppikEtJelWsKQERbLQsYm0IOOnzhwE9xYZC7x8txNnFC9wJNOkPferQI4vD4ZkC0mDyrOA==",
"hasInstallScript": true,
"dependencies": {
"debug": "^4.1.1",
"get-symbol-from-current-process-h": "^1.0.2",
"node-addon-api": "^3.0.0",
"node-gyp-build": "^4.2.1"
},
"engines": {
"node": ">= 10.0"
}
},
"node_modules/ref-napi/node_modules/debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
"dependencies": {
"ms": "2.1.2"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/ref-napi/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/ref-struct-di": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz",
"integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==",
"dependencies": {
"debug": "^3.1.0"
}
},
"node_modules/ref-struct-di/node_modules/debug": {
"version": "3.2.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
"dependencies": {
"ms": "^2.1.1"
}
},
"node_modules/ref-struct-di/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/requires-port": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
......
src_out.js
View file @ 4e366aaf
This diff is collapsed.
test_cookie.py
View file @ 4e366aaf
This diff is collapsed.
wn/Dockerfile
View file @ 4e366aaf
......@@ -10,11 +10,14 @@ RUN apt install -y make g++
COPY src/package.json /home/opt/lcc-wn/package.json
COPY src/package-lock.json /home/opt/lcc-wn/package-lock.json
RUN cd /home/opt/lcc-wn/ && npm i
RUN npm i ffi-napi
COPY src/src.js /home/opt/lcc-wn/src.js
COPY src/src2.js /home/opt/lcc-wn/src2.js
COPY src/jsdom /home/opt/lcc-wn/jsdom
COPY src/js_dom_run.js /home/opt/lcc-wn/js_dom_run.js
RUN npm i sync-request
COPY src/httpUtil.js /home/opt/lcc-wn/httpUtil.js
COPY src/tls-client-linux-ubuntu-amd64-v1.7.2.so /home/opt/lcc-wn/tls-client-linux-ubuntu-amd64-v1.7.2.so
COPY src/main.py /home/opt/lcc-wn/main.py
......
wn/httpUtil.js 0 → 100644
View file @ 4e366aaf
const ffi = require('ffi-napi');
// 判断设备是否为linux
const os = require('os');
if (os.platform() === 'linux') {
path = './tls-client-linux-ubuntu-amd64-v1.7.2.so';
} else {
path = './tls-client-arm64.dylib'
}
// load the tls-client shared package for your OS you are currently running your nodejs script (i'm running on mac)
const tlsClientLibrary = ffi.Library(path, {
'request': ['string', ['string']],
'getCookiesFromSession': ['string', ['string']],
'addCookiesToSession': ['string', ['string']],
'freeMemory': ["void", ['string']],
'destroyAll': ['string', []],
'destroySession': ['string', ['string']]
});
function requestSync(method, url, headers, proxy, data) {
const requestPayload = {
"tlsClientIdentifier": "chrome_117",
"followRedirects": true,
"insecureSkipVerify": false,
"withoutCookieJar": false,
"withDefaultCookieJar": false,
"isByteRequest": false,
"catchPanics": false,
"withDebug": false,
"forceHttp1": false,
"withRandomTLSExtensionOrder": false,
"timeoutSeconds": 5,
"timeoutMilliseconds": 0,
"sessionId": "my-session-id",
"proxyUrl": "",
"isRotatingProxy": false,
"certificatePinningHosts": {},
"headers": headers,
"headerOrder": [
"accept",
"user-agent",
"accept-encoding",
"accept-language"
],
"requestUrl": url,
"requestMethod": method,
"requestBody": data,
"requestCookies": []
}
// call the library with the requestPayload as string
const response = tlsClientLibrary.request(JSON.stringify(requestPayload));
// convert response string to json
const responseObject = JSON.parse(response)
console.log(responseObject)
tlsClientLibrary.freeMemory(responseObject.id)
const payload = {
sessionId: 'my-session-id',
url: "https://microsoft.com",
}
const cookiesResponse = tlsClientLibrary.getCookiesFromSession(JSON.stringify(payload))
const cookiesInSession = JSON.parse(cookiesResponse)
const destroySessionPayload = {
sessionId: 'my-session-id',
}
const destroySessionResponse = tlsClientLibrary.destroySession(JSON.stringify(destroySessionPayload))
const destroySessionResponseParsed = JSON.parse(destroySessionResponse)
// console.log(destroySessionResponseParsed)
return responseObject
}
module.exports = requestSync
\ No newline at end of file
wn/src/env.json
View file @ 4e366aaf
This diff is collapsed.
wn/src/httpUtil.js 0 → 100644
View file @ 4e366aaf
const ffi = require('ffi-napi');
// 判断设备是否为linux
const os = require('os');
if (os.platform() === 'linux') {
path = './tls-client-linux-ubuntu-amd64-v1.7.2.so';
} else {
path = './tls-client-arm64.dylib'
}
// load the tls-client shared package for your OS you are currently running your nodejs script (i'm running on mac)
const tlsClientLibrary = ffi.Library(path, {
'request': ['string', ['string']],
'getCookiesFromSession': ['string', ['string']],
'addCookiesToSession': ['string', ['string']],
'freeMemory': ["void", ['string']],
'destroyAll': ['string', []],
'destroySession': ['string', ['string']]
});
function requestSync(method, url, headers, proxy, data) {
const requestPayload = {
"tlsClientIdentifier": "chrome_117",
"followRedirects": true,
"insecureSkipVerify": false,
"withoutCookieJar": false,
"withDefaultCookieJar": false,
"isByteRequest": false,
"catchPanics": false,
"withDebug": false,
"forceHttp1": false,
"withRandomTLSExtensionOrder": false,
"timeoutSeconds": 5,
"timeoutMilliseconds": 0,
"sessionId": "my-session-id",
"proxyUrl": "",
"isRotatingProxy": false,
"certificatePinningHosts": {},
"headers": headers,
"headerOrder": [
"accept",
"user-agent",
"accept-encoding",
"accept-language"
],
"requestUrl": url,
"requestMethod": method,
"requestBody": data,
"requestCookies": []
}
// call the library with the requestPayload as string
const response = tlsClientLibrary.request(JSON.stringify(requestPayload));
// convert response string to json
const responseObject = JSON.parse(response)
console.log(responseObject)
tlsClientLibrary.freeMemory(responseObject.id)
const payload = {
sessionId: 'my-session-id',
url: "https://microsoft.com",
}
const cookiesResponse = tlsClientLibrary.getCookiesFromSession(JSON.stringify(payload))
const cookiesInSession = JSON.parse(cookiesResponse)
const destroySessionPayload = {
sessionId: 'my-session-id',
}
const destroySessionResponse = tlsClientLibrary.destroySession(JSON.stringify(destroySessionPayload))
const destroySessionResponseParsed = JSON.parse(destroySessionResponse)
// console.log(destroySessionResponseParsed)
return responseObject
}
module.exports = requestSync
\ No newline at end of file
wn/src/js_dom_run.js
View file @ 4e366aaf
This source diff could not be displayed because it is too large. You can view the blob instead.
wn/src/js_dom_run1.js
View file @ 4e366aaf
This diff is collapsed.
wn/src/main.py
View file @ 4e366aaf
......@@ -6,14 +6,13 @@ import subprocess
import time
import os
import loguru
# import requests
import requests
import json
# import tls_client
from bson import ObjectId
import hashlib
from pymongo import MongoClient
from loguru import logger
from curl_cffi import requests
def get_random_location():
return ''
......@@ -221,32 +220,31 @@ while True:
# proxy = f'http://ynfflcc2023.hotmail.com:gdcfh9@69.30.227.194:{random.randint(20001, 20250)}'
proxy2 = f'http://user-uni003-region-us-sessid-{random.randint(1000,5000)}-sesstime-5-keep-true:q39CEBTs5A5YQXor@pr.roxlabs.cn:4600'
proxy = f'http://user-uni003-region-us-sessid-{random.randint(1000,5000)}-sesstime-5-keep-true:q39CEBTs5A5YQXor@pr.roxlabs.cn:4600'
print(proxy)
# proxy = f'http://user-uni003-region-us-sessid-{random.randint(1000,5000)}-sesstime-5-keep-true:q39CEBTs5A5YQXor@pr.roxlabs.cn:4600'
# proxy = f'http://unfflcc:76cc14-47b8dd-1f8ace-827836-0c740e@198.147.25.26:{int(random.randint(10000, 11149))}'
# proxy = 'http://user-unifflcc-region-us:q39CEBTs5A5YQXor@pr.roxlabs.cn:4600'
tlsV = f'chrome_{random.randint(105, 113)}'
# proxy = random.choice([proxy1, proxy2])
# proxy = get_proxy()['url']
proxy = get_proxy()['url']
logger.info(f"success:{success} proxy: {proxy} ")
tlsV = 'chrome_112'
session = requests.Session(impersonate='chrome110')
# session = tls_client.Session(tlsV)
# session.timeout_seconds = 15
# session = requests.session()
session.proxies = {
'http': proxy,
'https': proxy,
}
# session.verify = False
if platform.uname().system == 'Darwin':
session.proxies = {
'http': 'http://127.0.0.1:7890',
'https': 'http://127.0.0.1:7890',
}
# session = requests.Session(impersonate='chrome110')
# # session = tls_client.Session(tlsV)
# # session.timeout_seconds = 15
# # session = requests.session()
# session.proxies = {
# 'http': proxy,
# 'https': proxy,
# }
# # session.verify = False
# if platform.uname().system == 'Darwin':
# session.proxies = {
# 'http': 'http://127.0.0.1:7890',
# 'https': 'http://127.0.0.1:7890',
# }
selectId = random.choice(list(envs))
# selectId = 'a74048568ce5f6140df6d956'
# selectId = '197d94c3099d110aef4e0a52'
if usedTime > 5:
# c = session.get('https://www.southwest.com/assets/app/scripts/swa-common.js').text
response = requests.post('http://52.52.23.116/tls/forward',
......@@ -304,17 +302,18 @@ setTimeout(r=> {
# f.write(code + '\n\n' + get_random_location() + ';;' + c)
code1 = open('js_dom_run.js','r').read()
# with open('js_dom_run1.js', 'w', encoding='utf-8') as f:
# f.write(code1 + '\n\n' + c)
with open('js_dom_run1.js', 'w', encoding='utf-8') as f:
f.write(code1 + '\n\n' + c)
t1 = time.time()
# token = list(db.token_shape.aggregate( [ {'$match': {'update_time':{'$gte': int(time.time() * 1000) - 60*60*12}}}, { '$sample': { 'size': 1 } } ] ))[0]['_id']
# token = '11'
# r = requests.get('http://127.0.0.1:5000/getinfo').json()['message']
# r = requests.get('http://127.0.0.1:3001').json()
logger.info("start execute js")
# p = subprocess.Popen(['node', 'js_dom_run1.js', selectId, '11'], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
# p = subprocess.Popen(['node', 'src_run_chrome_105.js', selectId, '11', token], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
# p = subprocess.Popen(['node', 'src_run_company_chrome.js', selectId, '11', token], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
logger.info("start execute js")
p = subprocess.Popen(['node', 'src_run1.js', selectId, '11', token, proxy], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, err = p.communicate(timeout=10)
......@@ -330,8 +329,8 @@ setTimeout(r=> {
# )
usedTime += 1
# print(r)
xhr_url = r.pop('xhr_url')
xhr_res = r.pop('xhr_res')
# xhr_url = r.pop('xhr_url')
# xhr_res = r.pop('xhr_res')
# print('xhr_url', xhr_url)
# print('xhr_res', xhr_res)
headers.update(r)
......
wn/src/src2.js
View file @ 4e366aaf
This diff is collapsed.
wn/src_run1.js
View file @ 4e366aaf
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment