Commit 6aa08327 authored by wangmingming's avatar wangmingming

init

parents
## tls-requests
仿照python request 的golang tls 库
模拟ja3 指纹
\ No newline at end of file
package easyreq
import (
"github.com/wmm1996528/requests/models"
"github.com/wmm1996528/requests/url"
"net/http"
)
func Request(method, rawurl string, req *url.Request) (*models.Response, error) {
session := NewSession()
return session.Request(method, rawurl, req)
}
func Get(rawurl string, req *url.Request) (*models.Response, error) {
return Request(http.MethodGet, rawurl, req)
}
func Post(rawurl string, req *url.Request) (*models.Response, error) {
return Request(http.MethodPost, rawurl, req)
}
func Options(rawurl string, req *url.Request) (*models.Response, error) {
return Request(http.MethodOptions, rawurl, req)
}
func Head(rawurl string, req *url.Request) (*models.Response, error) {
req.AllowRedirects = false
return Request(http.MethodHead, rawurl, req)
}
func Put(rawurl string, req *url.Request) (*models.Response, error) {
return Request(http.MethodPut, rawurl, req)
}
func Patch(rawurl string, req *url.Request) (*models.Response, error) {
return Request(http.MethodPatch, rawurl, req)
}
func Delete(rawurl string, req *url.Request) (*models.Response, error) {
return Request(http.MethodDelete, rawurl, req)
}
func Connect(rawurl string, req *url.Request) (*models.Response, error) {
return Request(http.MethodConnect, rawurl, req)
}
func Trace(rawurl string, req *url.Request) (*models.Response, error) {
return Request(http.MethodTrace, rawurl, req)
}
module github.com/wmm1996528/requests
go 1.20
require (
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/bitly/go-simplejson v0.5.0 // indirect
github.com/bogdanfinn/fhttp v0.5.22 // indirect
github.com/bogdanfinn/tls-client v1.3.12 // indirect
github.com/bogdanfinn/utls v1.5.16 // indirect
github.com/klauspost/compress v1.16.5 // indirect
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect
golang.org/x/crypto v0.9.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
)
package models
import (
"encoding/json"
"errors"
"fmt"
"github.com/bitly/go-simplejson"
http "github.com/bogdanfinn/fhttp"
"github.com/wmm1996528/requests/url"
"io"
)
// Response结构体
type Response struct {
Url string
Headers url.Header
Cookies []*http.Cookie
Text string
Content []byte
Body io.ReadCloser
StatusCode int
History []*Response
Request *url.Request
}
// 使用自带库JSON解析
func (res *Response) Json() (map[string]interface{}, error) {
js := make(map[string]interface{})
err := json.Unmarshal(res.Content, &js)
return js, err
}
// 使用go-simplejson解析
func (res *Response) SimpleJson() (*simplejson.Json, error) {
return simplejson.NewFromReader(res.Body)
}
// 状态码是否错误
func (res *Response) RaiseForStatus() error {
var err error
if res.StatusCode >= 400 && res.StatusCode < 500 {
err = errors.New(fmt.Sprintf("%d Client Error", res.StatusCode))
} else if res.StatusCode >= 500 && res.StatusCode < 600 {
err = errors.New(fmt.Sprintf("%d Server Error", res.StatusCode))
}
return err
}
package easyreq
import (
"bytes"
"encoding/json"
http "github.com/bogdanfinn/fhttp"
"github.com/bogdanfinn/fhttp/cookiejar"
"github.com/wmm1996528/requests/models"
"github.com/wmm1996528/requests/tls"
"github.com/wmm1996528/requests/url"
"io"
)
func NewSession() *Session {
return &Session{}
}
type Session struct {
Params *url.Params
Headers *url.Header
Cookies *cookiejar.Jar
Auth []string
Proxies string
Verify bool
Cert []string
Ja3 string
MaxRedirects int
request *url.Request
tlsVersion int
}
// 预请求处理
// http请求方式基础函数
func (s *Session) Request(method, rawurl string, request *url.Request) (*models.Response, error) {
if request == nil {
request = url.NewRequest()
}
resp, err := s.Do(method, request)
if err != nil {
return nil, err
}
return resp, nil
}
// get请求方式
func (s *Session) Get(rawurl string, req *url.Request) (*models.Response, error) {
return s.Request(http.MethodGet, rawurl, req)
}
// post请求方式
func (s *Session) Post(rawurl string, req *url.Request) (*models.Response, error) {
return s.Request(http.MethodPost, rawurl, req)
}
// options请求方式
func (s *Session) Options(rawurl string, req *url.Request) (*models.Response, error) {
return s.Request(http.MethodOptions, rawurl, req)
}
// head请求方式
func (s *Session) Head(rawurl string, req *url.Request) (*models.Response, error) {
return s.Request(http.MethodHead, rawurl, req)
}
// put请求方式
func (s *Session) Put(rawurl string, req *url.Request) (*models.Response, error) {
return s.Request(http.MethodPut, rawurl, req)
}
// patch请求方式
func (s *Session) Patch(rawurl string, req *url.Request) (*models.Response, error) {
return s.Request(http.MethodPatch, rawurl, req)
}
// delete请求方式
func (s *Session) Delete(rawurl string, req *url.Request) (*models.Response, error) {
return s.Request(http.MethodDelete, rawurl, req)
}
// connect请求方式
func (s *Session) Connect(rawurl string, req *url.Request) (*models.Response, error) {
return s.Request(http.MethodConnect, rawurl, req)
}
func (s *Session) Do(method string, request *url.Request) (*models.Response, error) {
client := tls.NewClient(request.TlsProfile)
request.Method = method
preq, err := s.PreRequest(request)
if err == nil {
return nil, err
}
do, err := client.Do(preq)
if err != nil {
return nil, err
}
defer do.Body.Close()
return s.PreResponse(request, do)
}
func (s *Session) PreRequest(request *url.Request) (*http.Request, error) {
datas, err := s.PreData(request)
if err != nil {
return nil, err
}
req, err := http.NewRequest(request.Method, request.Url, bytes.NewBuffer(datas))
if err != nil {
return nil, err
}
headers := request.Headers.GetAll()
for k, v := range headers {
req.Header.Set(k, v)
}
return req, nil
}
// PreData 解析post参数
func (s *Session) PreData(request *url.Request) ([]byte, error) {
if request.Json != nil {
return json.Marshal(request.Json)
}
if request.Data != nil {
return []byte(request.Data.Encode()), nil
}
return []byte{}, nil
}
// PreResponse 处理请求返回
func (s *Session) PreResponse(request *url.Request, do *http.Response) (*models.Response, error) {
reader := http.DecompressBody(do)
rb, _ := io.ReadAll(reader)
redirectURL, err := do.Location()
redirectUrl := ""
if err == nil {
// 没有错误时从跳转的location 获取url
redirectUrl = redirectURL.String()
}
resp := &models.Response{
Url: redirectUrl,
Headers: url.Header(do.Header),
Cookies: do.Cookies(),
Text: string(rb),
Content: rb,
Body: reader,
StatusCode: do.StatusCode,
Request: request,
}
return resp, nil
}
package tls
import (
tls_client "github.com/bogdanfinn/tls-client"
"log"
)
func NewClient(tlsProfile int) tls_client.HttpClient {
var tlsConfig tls_client.ClientProfile
switch tlsProfile {
case Chrome112:
tlsConfig = tls_client.Chrome_112
case Chrome111:
tlsConfig = tls_client.Chrome_111
case Chrome110:
tlsConfig = tls_client.Chrome_110
case Chrome109:
tlsConfig = tls_client.Chrome_109
case Chrome108:
tlsConfig = tls_client.Chrome_108
case Chrome107:
tlsConfig = tls_client.Chrome_107
case Chrome106:
tlsConfig = tls_client.Chrome_106
case Chrome105:
tlsConfig = tls_client.Chrome_105
case Ios16:
tlsConfig = tls_client.Safari_IOS_16_0
case Ios15:
tlsConfig = tls_client.Safari_IOS_15_5
default:
tlsConfig = tls_client.Chrome_112
}
jar := tls_client.NewCookieJar()
options := []tls_client.HttpClientOption{
tls_client.WithTimeoutSeconds(15),
tls_client.WithClientProfile(tlsConfig),
tls_client.WithCookieJar(jar), // create cookieJar instance and pass it as argument
}
client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...)
if err != nil {
log.Println(err)
}
return client
}
package tls
// * 要使用的tls版本
const (
Chrome112 = 1 + iota
Chrome111
Chrome110
Chrome109
Chrome108
Chrome107
Chrome106
Chrome105
Ios16
Ios15
)
package url
import (
"errors"
http "github.com/bogdanfinn/fhttp"
"github.com/bogdanfinn/fhttp/cookiejar"
"net/url"
"strings"
)
func NewCookies() *cookiejar.Jar {
cookies, _ := cookiejar.New(nil)
return cookies
}
func ParseCookies(rawurl, cookies string) *cookiejar.Jar {
c := NewCookies()
cookieList := strings.Split(cookies, ";")
urls, _ := url.Parse(rawurl)
for _, cookie := range cookieList {
cookie = strings.TrimSpace(cookie)
if cookie == "" {
continue
}
keyValue := strings.SplitN(cookie, "=", 2)
if len(keyValue) != 2 {
panic(errors.New("该字符串不符合Cookies标准"))
}
key := keyValue[0]
value := keyValue[1]
c.SetCookies(urls, []*http.Cookie{&http.Cookie{
Name: key,
Value: value,
}})
}
return c
}
package url
import (
"errors"
"net/textproto"
"strings"
)
const HeaderOrderKey = "Header-Order:"
// PHeaderOrderKey is a magic Key for setting http2 pseudo header order.
// If the header is nil it will use regular GoLang header order.
// Valid fields are :authority, :method, :path, :scheme
const PHeaderOrderKey = "PHeader-Order:"
// * 请求header 部分
type Header Common
func (h Header) Add(key, value string) {
textproto.MIMEHeader(h).Add(key, value)
}
// Set sets the header entries associated with Key to the
// single element value. It replaces any existing Values
// associated with Key. The Key is case insensitive; it is
// canonicalized by textproto.CanonicalMIMEHeaderKey.
// To use non-canonical keys, assign to the map directly.
func (h Header) Set(key, value string) {
textproto.MIMEHeader(h).Set(key, value)
}
// Get gets the first value associated with the given Key. If
// there are no Values associated with the Key, Get returns "".
// It is case insensitive; textproto.CanonicalMIMEHeaderKey is
// used to canonicalize the provided Key. To use non-canonical keys,
// access the map directly.
func (h Header) Get(key string) string {
return textproto.MIMEHeader(h).Get(key)
}
func (h Header) GetAll() map[string]string {
res := make(map[string]string)
for k, _ := range h {
res[k] = h.Get(k)
}
return res
}
// Values returns all Values associated with the given Key.
// It is case insensitive; textproto.CanonicalMIMEHeaderKey is
// used to canonicalize the provided Key. To use non-canonical
// keys, access the map directly.
// The returned slice is not a copy.
func (h Header) Values(key string) []string {
return textproto.MIMEHeader(h).Values(key)
}
// get is like Get, but Key must already be in CanonicalHeaderKey form.
func (h Header) get(key string) string {
if v := h[key]; len(v) > 0 {
return v[0]
}
return ""
}
// has reports whether h has the provided Key defined, even if it's
// set to 0-length slice.
func (h Header) has(key string) bool {
_, ok := h[key]
return ok
}
// Del deletes the Values associated with Key.
// The Key is case insensitive; it is canonicalized by
// CanonicalHeaderKey.
func (h Header) Del(key string) {
textproto.MIMEHeader(h).Del(key)
}
// Clone returns a copy of h or nil if h is nil.
func (h Header) Clone() Header {
if h == nil {
return nil
}
// Find total number of Values.
nv := 0
for _, vv := range h {
nv += len(vv)
}
sv := make([]string, nv) // shared backing array for headers' Values
h2 := make(Header, len(h))
for k, vv := range h {
n := copy(sv, vv)
h2[k] = sv[:n:n]
sv = sv[n:]
}
return h2
}
// 初始化Headers结构体
func NewHeaders() *Header {
headers := &Header{}
(*headers)[PHeaderOrderKey] = (*headers)[PHeaderOrderKey]
return headers
}
// 解析Headers字符串为结构体
func ParseHeaders(headers string) *Header {
h := Header{}
headerOrder := []string{}
lines := strings.Split(headers, "\n")
for _, header := range lines {
header = strings.TrimSpace(header)
if header == "" || strings.Index(header, ":") == 0 || strings.Index(header, "/") == 0 || strings.Index(header, "#") == 0 {
continue
}
keyValue := strings.SplitN(header, ":", 2)
if len(keyValue) != 2 {
panic(errors.New("该字符串不符合http头部标准!"))
}
key := keyValue[0]
value := keyValue[1]
h.Set(key, value)
headerOrder = append(headerOrder, strings.ToLower(key))
}
h[HeaderOrderKey] = headerOrder
return &h
}
package url
// Values结构体
type Json struct {
values map[string]interface{}
}
func NewJson() *Json {
value := make(map[string]interface{})
return &Json{values: value}
}
package url
import (
"github.com/wmm1996528/requests/utils"
"strings"
)
// 查询下标
func SearchStrings(str []string, substr string) int {
for index, value := range str {
if value == substr {
return index
}
}
return -1
}
// 解析params字符串为Params结构体
func ParseParams(params string) *Params {
p := NewParams()
if params == "" {
return p
}
for _, l := range strings.Split(params, "&") {
value := strings.SplitN(l, "=", 2)
if len(value) == 2 {
p.Add(value[0], value[1])
}
}
return p
}
// 初始化Params结构体
func NewParams() *Params {
return &Params{}
}
// Params结构体
type Params struct {
params []map[string][]string
indexKey []string
}
// 设置Params参数
func (p *Params) Set(key, value string) {
pm := map[string][]string{
key: []string{value},
}
index := SearchStrings(p.indexKey, key)
if len(p.indexKey) == 0 || index == -1 {
p.params = append(p.params, pm)
p.indexKey = append(p.indexKey, key)
} else {
p.params[index] = pm
}
}
// 获取Params参数值
func (p *Params) Get(key string) string {
if len(p.params) != 0 {
index := SearchStrings(p.indexKey, key)
if index != -1 {
return p.params[index][key][0]
}
}
return ""
}
// 添加Params参数
func (p *Params) Add(key, value string) bool {
index := SearchStrings(p.indexKey, key)
if len(p.indexKey) == 0 || index == -1 {
p.Set(key, value)
} else {
p.params[index][key] = append(p.params[index][key], value)
}
return true
}
// 删除Params参数
func (p *Params) Del(key string) bool {
index := SearchStrings(p.indexKey, key)
if len(p.indexKey) == 0 || index == -1 {
return false
}
p.params = append(p.params[:index], p.params[index+1:]...)
p.indexKey = append(p.indexKey[:index], p.indexKey[index+1:]...)
return true
}
// 获取Params的所有Key
func (p *Params) Keys() []string {
return p.indexKey
}
// Params结构体转字符串
func (p *Params) Encode() string {
text := []string{}
for index, key := range p.indexKey {
item := p.params[index][key]
for _, value := range item {
text = append(text, utils.EncodeURIComponent(key)+"="+utils.EncodeURIComponent(value))
}
}
return strings.Join(text, "&")
}
package url
import (
"github.com/bogdanfinn/fhttp/cookiejar"
"time"
)
func NewRequest() *Request {
return &Request{
AllowRedirects: true,
Verify: true,
}
}
type Request struct {
Params *Params
Headers *Header
Cookies *cookiejar.Jar
Data *Values
Json map[string]interface{}
Body string
Auth []string
Timeout time.Duration
AllowRedirects bool
Proxies string
Verify bool
ForceHTTP1 bool
TlsProfile int
Method string
Url string
}
package url
import (
"github.com/wmm1996528/requests/utils"
"strings"
)
// 解析Values字符串为Values结构体
func ParseValues(params string) *Values {
p := NewValues()
if params == "" {
return p
}
for _, l := range strings.Split(params, "&") {
value := strings.SplitN(l, "=", 2)
if len(value) == 2 {
p.Add(value[0], value[1])
}
}
return p
}
// 解析Data字符串为Values结构体
func ParseData(data string) *Values {
return ParseValues(data)
}
// 初始化Values结构体
func NewValues() *Values {
return &Values{}
}
// 初始化Data结构体
func NewData() *Values {
return &Values{}
}
// Values结构体
type Values struct {
values []map[string][]string
indexKey []string
}
// 设置Values参数
func (v *Values) Set(key, value string) {
p := map[string][]string{
key: []string{value},
}
index := SearchStrings(v.indexKey, key)
if len(v.indexKey) == 0 || index == -1 {
v.values = append(v.values, p)
v.indexKey = append(v.indexKey, key)
} else {
v.values[index] = p
}
}
// 获取Values参数值
func (v *Values) Get(key string) string {
if len(v.values) != 0 {
index := SearchStrings(v.indexKey, key)
if index != -1 {
return v.values[index][key][0]
}
return ""
}
return ""
}
// 添加Values参数
func (v *Values) Add(key, value string) bool {
index := SearchStrings(v.indexKey, key)
if len(v.indexKey) == 0 || index == -1 {
v.Set(key, value)
} else {
v.values[index][key] = append(v.values[index][key], value)
}
return true
}
// 删除Values参数
func (v *Values) Del(key string) bool {
index := SearchStrings(v.indexKey, key)
if len(v.indexKey) == 0 || index == -1 {
return false
}
v.values = append(v.values[:index], v.values[index+1:]...)
v.indexKey = append(v.indexKey[:index], v.indexKey[index+1:]...)
return true
}
// 获取Values的所有Key
func (v *Values) Keys() []string {
return v.indexKey
}
// Values结构体转字符串
func (v *Values) Encode() string {
text := []string{}
for index, key := range v.indexKey {
item := v.values[index][key]
for _, value := range item {
text = append(text, utils.EncodeURIComponent(key)+"="+utils.EncodeURIComponent(value))
}
}
return strings.Join(text, "&")
}
package url
type Common map[string][]string
package utils
import (
"bytes"
"encoding/base32"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"errors"
"net/url"
"regexp"
"strconv"
"strings"
)
// 只接受string和[]byte类型
func stringAndByte(s interface{}) []byte {
var byte_s []byte
var err error
switch s.(type) {
case string:
byte_s = []byte(s.(string))
case []byte:
byte_s = s.([]byte)
default:
err = errors.New("Please check whether the type is string and []byte.")
panic(err)
}
return byte_s
}
// Hex编码
func HexEncode(s interface{}) []byte {
byte_s := stringAndByte(s)
dst := make([]byte, hex.EncodedLen(len(byte_s)))
n := hex.Encode(dst, byte_s)
return dst[:n]
}
// Hex解码
func HexDecode(s interface{}) []byte {
byte_s := stringAndByte(s)
dst := make([]byte, hex.DecodedLen(len(byte_s)))
n, err := hex.Decode(dst, byte_s)
if err != nil {
panic(err)
}
return dst[:n]
}
// URI编码
func EncodeURIComponent(s interface{}) string {
byte_s := stringAndByte(s)
es := url.QueryEscape(string(byte_s))
es = strings.ReplaceAll(es, "+", "%20")
es = strings.ReplaceAll(es, "%21", "!")
es = strings.ReplaceAll(es, "%27", "'")
es = strings.ReplaceAll(es, "%28", "(")
es = strings.ReplaceAll(es, "%29", ")")
es = strings.ReplaceAll(es, "%2A", "*")
return es
}
// URI解码
func DecodeURIComponent(s interface{}) string {
byte_s := stringAndByte(s)
t, err := url.QueryUnescape(strings.ReplaceAll(strings.ReplaceAll(string(byte_s), "+", "%2B"), "%20", "+"))
if err != nil {
panic(err)
}
return t
}
// URI编码
func EncodeURI(s interface{}) string {
byte_s := stringAndByte(s)
es := EncodeURIComponent(string(byte_s))
ss := "!#$&'()*+,-./:=?@_~"
for i := 0; i < len(ss); i++ {
es = strings.ReplaceAll(es, "%"+strings.ToUpper(string(HexEncode(string(ss[i])))), string(ss[i]))
}
return strings.ReplaceAll(es, "%3B", ";")
}
// URI解码
func DecodeURI(s interface{}) string {
byte_s := stringAndByte(s)
es := string(byte_s)
ss := "!#$&'()*+,-./:=?@_~"
for i := 0; i < len(ss); i++ {
es = strings.ReplaceAll(es, "%"+strings.ToUpper(string(HexEncode(string(ss[i])))), "$"+"%"+strings.ToUpper(string(HexEncode(string(ss[i]))))+"$")
}
es = DecodeURIComponent(es)
for i := 0; i < len(ss); i++ {
es = strings.ReplaceAll(es, "$"+string(ss[i])+"$", "%"+strings.ToUpper(string(HexEncode(string(ss[i])))))
}
return es
}
// Base32编码
func Base32Encode(s interface{}) string {
byte_s := stringAndByte(s)
return base32.StdEncoding.EncodeToString(byte_s)
}
// Base32解码
func Base32Decode(s interface{}) string {
byte_s := stringAndByte(s)
str, err := base32.StdEncoding.DecodeString(string(byte_s))
if err != nil {
panic(err)
}
return string(str)
}
// Base64编码
func Btoa(s interface{}) string {
byte_s := stringAndByte(s)
return base64.StdEncoding.EncodeToString(byte_s)
}
// Base64编码,同上
func Base64Encode(s interface{}) string {
byte_s := stringAndByte(s)
return Btoa(byte_s)
}
// Base64解码
func Atob(s interface{}) string {
byte_s := stringAndByte(s)
str, err := base64.StdEncoding.DecodeString(string(byte_s))
if err != nil {
panic(err)
}
return string(str)
}
// Base64解码,同上
func Base64Decode(s interface{}) string {
byte_s := stringAndByte(s)
return Atob(byte_s)
}
// 中文转Unicode
func Escape(s interface{}) string {
byte_s := stringAndByte(s)
str := string(byte_s)
es := ""
for _, s := range str {
switch {
case s >= '0' && s <= '9':
es += string(s)
case s >= 'a' && s <= 'z':
es += string(s)
case s >= 'A' && s <= 'Z':
es += string(s)
case strings.Contains("*+-./@_", string(s)):
es += string(s)
case int(s) <= 127:
es += "%" + strings.ToUpper(string(HexEncode(string(s))))
case int(s) >= 128:
es += strings.ReplaceAll(strings.ReplaceAll(strconv.QuoteToASCII(string(s)), "\"", ""), "\\u", "%u")
}
}
return es
}
// Unicode转中文
func UnEscape(s interface{}) string {
byte_s := stringAndByte(s)
str := string(byte_s)
re, _ := regexp.Compile("(%u)[0-9a-zA-Z]{4}")
str = re.ReplaceAllStringFunc(str, func(st string) string {
bs, _ := hex.DecodeString(strings.ReplaceAll(st, "%u", ""))
r := uint16(0)
binary.Read(bytes.NewReader(bs), binary.BigEndian, &r)
return string(r)
})
str = DecodeURIComponent(str)
return str
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment