librauee / reptile Goto Github PK

import requests
from pymongo import MongoClient
import time
import random

class Typhoon:
    def __init__(self):
        self.user_agent = [
                           "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
                           "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
                           "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0",
                           "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko",
                           "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
    ]

        self.base_url = 'http://www.wztf121.com/data/complex/{}.json'
        self.headers = {
            'Cookie': '_gscu_1378142123=65572018r5on4x80; _gscbrs_1378142123=1; vjuids=30469f88b.16c835d32ea.0.8062809782e9b; vjlast=1565572019.1565572019.30; Hm_lvt_e592d6befa4f9918e6496980d22c5649=1565572019; Wa_lvt_1=1565572019; Wa_lpvt_1=1565576034; _gscs_1378142123=65572018v2ofkf80|pv:8; Hm_lpvt_e592d6befa4f9918e6496980d22c5649=1565576061',
            'Host': 'www.wztf121.com',
            'Referer': 'http://www.wztf121.com/history.html',
            'User-Agent': random.choice(self.user_agent)
        }

        self.client = MongoClient()
        self.db     = self.client.typhoon



    def get_year(self):
        year_list = []
        years_url = self.base_url.format('years')

        r = requests.get(years_url, headers = self.headers)
        years = r.json()

        for year in years:
            year_list.append(year['year'])

        print('以获取所有台风记录的年份')
        return year_list

    def get_tf_list(self):
        tf_list = []
        year_list = self.get_year()

        for year in year_list:
            url = self.base_url.format(year)

            r = requests.get(url, headers = self.headers)
            tfs = r.json()

            for tf in tfs:
                tfbh = tf['tfbh']
                tf_list.append(tfbh)

            time.sleep(random.random())

        print('已获得所有台风的编号，格式为 年份 + 次序')
        return tf_list

    def get_tf_detail(self):
        tf_list = self.get_tf_list()
        count = 1
        for tf in tf_list:
            tf_url = self.base_url.format(tf)
            r = requests.get(tf_url, headers = self.headers)
            tf_detail = r.json()

            begin_time = tf_detail[0]['begin_time']
            ename      = tf_detail[0]['ename']
            end_time   = tf_detail[0]['end_time']
            name       = tf_detail[0]['name']
            points     = tf_detail[0]['points']

            for point in points:
                latitude  = point['latitude']
                longitude = point['longitude']
                power     = point['power']
                speed     = point['speed']
                pressure  = point['pressure']
                strong    = point['strong']
                real_time = point['time']

                detail = {
                    'name': name,
                    'ename': ename,
                    'latitude': latitude,
                    'longitude': longitude,
                    'power': power,
                    'speed': speed,
                    'pressure': pressure,
                    'strong': strong,
                    'time': real_time,
                }
                self.db['detail'].insert_one(detail)


            time.sleep(5 * random.random())
            tf_info = {
                'name': name,
                'ename': ename,
                'begin_time': begin_time,
                'end_time': end_time,
            }

            self.db['info'].insert_one(tf_info)
            print('已存入第{}条台风详细信息!'.format(count))
            count += 1

                
            
        
tfcraw = Typhoon()
tfcraw.get_tf_detail()