coder-hxl / x-crawl Goto Github PK

import xCrawl from 'x-crawl';

  async crawl(path: string) {
    // 2.创建一个爬虫实例
    const myXCrawl = xCrawl();
    myXCrawl.crawlPage(path).then(res => {
      const { browser, page } = res.data;
      console.log(page);
      // 关闭浏览器
      browser.close();
    });
  }

import xCrawl from 'x-crawl'

// Application instance configuration
const testXCrawl = xCrawl({
  proxy: {
    urls: [
      'https://127.0.0.1:7890'
    ],
    switchByErrorCount: 1,
    switchByHttpStatus: [401, 403]
  }
})

// Advanced configuration
testXCrawl
  .crawlPage({
    targets: [
      'https://www.google.com',
      {
        url: 'https://www.google.com',
        proxy: { urls: ['http://127.0.0.1:7890'] }
      }
    ],
    maxRetry: 3,
    proxy: {
      urls: [
        'http://127.0.0.1:7890',
        'http://127.0.0.1:7890'
      ],
      switchByErrorCount: 1,
      switchByHttpStatus: [401, 403]
    }
  })
  .then((res) => {})

const pageSize = 30;
    const sort = 'time';
    const rating = 'all';
    const filter = 'all';
    const mode = 'list';
    const url = `https://${type}.${this.BASE_URL}/people/${uid}/${status}`;
    const request: CrawlDataDetailTargetConfig = {
      url,
      method: "GET",
      params: {
        start: (page - 1) * pageSize,
        sort,
        rating,
        filter,
        mode,
      },
      data: {},
      timeout: 30000,
      maxRetry: 0
    };

    const crawler = Crawl({
      enableRandomFingerprint: true
    });
    const res = await crawler.crawlData(request);

const post = {
    url: "https://glxy.mot.gov.cn/company/getCompanyAptitude.do",
    method: "post",
    headers: {
      "Content-Type": "application/x-www-form-urlencoded",
    },
    data: "page=1",
  };
  let res = await myXCrawl.crawlData(post);

import xCrawl from 'x-crawl';
import path from 'path';
import fs from 'fs/promises';

const bilibiliXCrawl = xCrawl({ mode: 'async' });
export function getCurrentDate(separator: string = '-') {


  // 调用 crawlFile API 爬取图片
  await bilibiliXCrawl.crawlFile({
    targets: [...urls],
    // storeDir: `./bilibili/${getCurrentDate('/')}`,
    storeDir,
  });
  // 关闭页面
  page.close();

  // 关闭浏览器
  browser.close();
}

pnpm i xxx

const targets = {
    url: "https://glxy.mot.gov.cn/company/getCompanyAptitude.do",
    method: "POST",
    headers: {
      "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
    },
    data: Qs.stringify({ type: 2 }),
  };
  const res = await myXCrawl.crawlData(targets);

npm i x-crawl

import { createCrawl } from 'x-crawl'

const crawlApp = createCrawl({ intervalTime: { max: 3000, min: 1000 } })
let aar =["https://suno-list.com/api/suno/validateLink?link=https%3A%2F%2Fsuno.com%2Fsong%2Fe8b369f2-e2fc-4152-b2aa-d29224d43041"]
    crawlApp.crawlData({ aar }).then((twoRes) => {
        // 处理接口链接
        console.log(twoRes);
        for(let one of twoRes){
            // 这里报错
        }
        console.log(downloadLinks);
    });

export function crawlFile(
  url: string | string[],
  config?: CrawlFileConfig,
  callback?: (result: CrawlFileSingleResult | CrawlFileSingleResult[]) => void
): Promise<CrawlFileSingleResult | CrawlFileSingleResult[]>

export interface CrawlFileConfig extends CrawlCommonConfig {
  outputDir?: string | string[] | null
  extension?: string | string[] | null
  fileName?: string | string[] | null
  intervalTime?: IntervalTime
  fingerprints?: DetailTargetFingerprintCommon[]
  headers?: AnyObject
  onCrawlItemComplete?: (result: CrawlFileSingleResult) => void
  onBeforeSaveItemFile?: (info: {
    id: number
    fileName: string
    filePath: string
    data: Buffer
  }) => Promise<Buffer>
}

我window上能正常使用的功能，在Linux上获取不到结果，也不报错，日志根据提示是完成的，我是通过fastify起api，debian 11 

fastify.post('/api/screenshoot', async function handler(request, reply) {
    const { url } = request.body
    if (!url) {
        return reply.send({ code: 0, msg: "url is required" })
    }
    try {
        const buffer = await screenshoot({ url })
        // 在发送非常规数据时。一定一定要指定响应数据类型
        reply.type("image/jpeg")
        return buffer
    } catch (error) {

    }
})

// screenshoot model
const res = await x.crawlPage({
            url,
            maxRetry: 10,
            viewport: { width: 1920, height: 1080 },
            // 为此次的目标统一设置指纹
            fingerprints: [
                // 设备指纹 1
                {
                    maxWidth: 1024,
                    maxHeight: 800,
                    platform: 'Windows',
                    mobile: 'random',
                    userAgent: {
                        value:
                            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36',
                        versions: [
                            {
                                name: 'Chrome',
                                // 浏览器版本
                                maxMajorVersion: 112,
                                minMajorVersion: 100,
                                maxMinorVersion: 20,
                                maxPatchVersion: 5000
                            },
                            {
                                name: 'Safari',
                                maxMajorVersion: 537,
                                minMajorVersion: 500,
                                maxMinorVersion: 36,
                                maxPatchVersion: 5000
                            }
                        ]
                    }
                },
                // 设备指纹 2
                {
                    platform: 'Windows',
                    mobile: 'random',
                    userAgent: {
                        value:
                            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Edg/91.0.864.59',
                        versions: [
                            {
                                name: 'Chrome',
                                maxMajorVersion: 91,
                                minMajorVersion: 88,
                                maxMinorVersion: 10,
                                maxPatchVersion: 5615
                            },
                            { name: 'Safari', maxMinorVersion: 36, maxPatchVersion: 2333 },
                            { name: 'Edg', maxMinorVersion: 10, maxPatchVersion: 864 }
                        ]
                    }
                },
                // 设备指纹 3
                {
                    platform: 'Windows',
                    mobile: 'random',
                    userAgent: {
                        value:
                            'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0',
                        versions: [
                            {
                                name: 'Firefox',
                                maxMajorVersion: 47,
                                minMajorVersion: 43,
                                maxMinorVersion: 10,
                                maxPatchVersion: 5000
                            }
                        ]
                    }
                }
            ]
        })

        const { browser, page } = res.data
        // // Get a screenshot of the rendered page
        const buffer = await page.screenshot({ path: `../uploads/${host}_${Date.now()}.png` })
        console.log('Screen capture is complete')

        if (buffer) {
            page.close()
        }
        // close brower
        // browser.close()

        return buffer

const crawler = Crawl();
const options = {
      'url': 'https://api.github.com/repos/xxx/xxx/releases',
      'method': 'GET',
      'headers': {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36',
        'X-GitHub-Api-Version': '2022-11-28',
        'Accept': 'application/json',
  'Authorization': 'Bearer tokenxxx'
      },
      'params': {
        'per_page': 1,
        'page': 1
      },
      'data': {},
      'timeout': 30000,
      'maxRetry': 0
    };
const res = await crawler.crawlData(options);