EADST

Python: Obtain Baidu Images Using Web Crawler

Python: Obtain Baidu Images Using Web Crawler.

Here is the main code.

# -- coding: utf-8 --
import os
import re
import time
import requests

class CarCollect():

def __init__(self, path='./name.txt'):
    self.num = 1
    self.class_number = 0
    self.line_list = []
    with open(path, encoding='utf-8') as file:
        self.line_list = [k.strip() for k in file.readlines()]
        self.class_number = int(self.line_list[0])
        self.line_list = self.line_list[1:]

def dowmload_picture(self, html, keyword, save_path):
    pic_url = re.findall('"objURL":"(.*?)",', html, re.S)  # get image url
    print('Finding keyword: ' + keyword + ' images, start downloading...')
    for each in pic_url:
        print('='*60)
        print('Downloading ' + keyword + ' number ' + str(self.num) + ' image, url: ' + str(each))
        try:
            if each:
                pic = requests.get(each, timeout=7)
                string = save_path + r'\\' + keyword + '_' + str(self.num) + '.jpg'
                if len(pic.content) > 10000: # img size > 10k
                    with open(string, 'wb') as fp:
                        fp.write(pic.content)
                        self.num += 1
        except BaseException:
            print('error, cannot download')
        if self.num > self.class_number:
            break

def __call__(self):
    headers = {
        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
        'Connection': 'keep-alive',
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0',
        'Upgrade-Insecure-Requests': '1'
    }
    session = requests.Session()
    session.headers = headers

    for word in self.line_list:
        # create a folder
        save_path = word + '_file'
        time_now = time.strftime("%Y%m%d_%H%M%S", time.localtime())
        save_path += "_" + time_now
        os.mkdir(save_path)
        # get images
        url = 'https://image.baidu.com/search/flip?tn=baiduimage&ie=utf-8&word=' + word + '&pn='
        image_number = 0
        self.num = 1
        while image_number < self.class_number:
            try:
                result = session.get(url + str(image_number), timeout=10, allow_redirects=False)
                self.dowmload_picture(result.text, word, save_path)
            except:
                print('Internet error')
            image_number += 60

if name == 'main': path = './keywords.txt' car_collect = CarCollect(path) car_collect() print('Done.')

Here is the text file, keywords.txt. The first line is the number we want to obtain from each keyword. The following lines are the keywords.

20
Dog
Cat

相关标签
About Me
XD
Goals determine what you are going to be.
Category
标签云
多进程 Google Sklearn Diagram Freesound Llama scipy Numpy Markdown VPN YOLO 论文 BF16 TTS Safetensors UNIX Datetime NLTK SQL Image2Text GIT AI Paper Django JSON Proxy NameSilo DeepStream 继承 Jev Data Jetson 第一性原理 Tiktoken hf CV GGML Heatmap CAM 长期记忆 Excel llama.cpp OCR 飞书 Firewall TensorRT Search C++ MD5 Vmess Baidu Nginx Pytorch Tensor Translation Miniforge ONNX Web RL Breakpoint Pillow v2ray DeepSeek Math Paddle printf HaggingFace CSV 版权 Algorithm 图标 Base64 Linux Bitcoin Hungarian Magnet 报税 Agent ResNet-50 EXCEL VGG-16 diffusers PIP Qwen2 LLM git-lfs RAR Website Knowledge Video 腾讯云 Card GoogLeNet Bin Hilton 算法题 图形思考法 XML 个人博客 论文速读 IndexTTS2 WebCrawler 多线程 torchinfo Land 域名 Bipartite mmap PyTorch Use uwsgi CUDA OpenCV Anaconda Clash VSCode InvalidArgumentError API网关 财报 Hotel TensorFlow Food 证件照 Python Color FP32 FastAPI 签证 净利润 公式 PDF v0.dev NLP CLAP QWEN FP16 Cloudreve BeautifulSoup Augmentation ChatGPT 搞笑 LoRA Pickle Logo FP8 强化学习 OpenAI BTC XGBoost tqdm Domain Qwen Github SVR Attention 云服务器 Interview LeetCode Template RGB Harness Crawler CTC Dataset UI Ubuntu PDB Ptyhon COCO SPIE GPTQ Rebuttal PyCharm ModelScope FlashAttention 自我改进 Mixtral Bert Distillation Disk SQLite API LLAMA transformers Plate Claude Statistics Docker Quantize WAN icon Animate SAM git Tracking 音频 Review 阿里云 GPT4 News Pandas Input Transformers 递归学习法 TSV Git HuggingFace uWSGI tar logger Zip Windows Password Shortcut LaTeX CEIR CC Vim Gemma Michelin Permission 关于博主 Quantization Random Plotly Jupyter Qwen2.5 FP64 Streamlit ms-swift 顶会 Conda
站点统计

本站现有博文340篇,共被浏览1019360次

本站已经建立2719天!

热门文章
文章归档
回到顶部