Danbooru爬虫

import requests
import os
import time

# 配置
ARTIST_NAME = "name"  # 目标艺术家名称
SAVE_FOLDER = f"danbooru_{ARTIST_NAME}_artworks"
BASE_URL = f"https://danbooru.donmai.us/posts.json?limit=200&tags={ARTIST_NAME}&search[is_deleted]=no"
PAGE_MAX = 1000
REQUEST_DELAY = 1

# 创建保存文件夹
if not os.path.exists(SAVE_FOLDER):
    os.makedirs(SAVE_FOLDER)
    print(f"创建文件夹: {SAVE_FOLDER}")

# 分页爬取并下载图片
for page in range(1, PAGE_MAX + 1):
    url = f"{BASE_URL}&page={page}"
    print(f"正在处理第 {page} 页...", flush=True)

    response = requests.get(url)
    if response.status_code != 200:
        print(f"第 {page} 页请求失败，状态码: {response.status_code}，终止爬取", flush=True)
        break

    data = response.json()
    if not data:
        print(f"第 {page} 页无数据，爬取完成", flush=True)
        break

    # 遍历下载图片
    for idx, artwork in enumerate(data):
        artwork_id = artwork.get("id")
        # 优先原图，无则用高清图
        image_url = artwork.get("file_url") or artwork.get("large_file_url")
        if not image_url:
            print(f"第 {page} 页第 {idx+1} 个作品无有效URL，跳过", flush=True)
            continue

        file_ext = artwork.get("file_ext", "jpg")
        save_path = os.path.join(SAVE_FOLDER, f"{ARTIST_NAME}_{artwork_id}.{file_ext}")

        # 避免重复下载
        if os.path.exists(save_path):
            print(f"文件 {os.path.basename(save_path)} 已存在，跳过", flush=True)
            continue

        try:
            img_response = requests.get(image_url, timeout=15)
            if img_response.status_code == 200:
                with open(save_path, "wb") as f:
                    f.write(img_response.content)
                print(f"成功下载: {os.path.basename(save_path)}", flush=True)
            else:
                print(f"下载失败: 作品ID {artwork_id}，状态码 {img_response.status_code}", flush=True)
        except Exception as e:
            print(f"下载异常: 作品ID {artwork_id}，错误: {str(e)}", flush=True)

    time.sleep(REQUEST_DELAY)

print(f"爬取结束！图片保存至: {os.path.abspath(SAVE_FOLDER)}", flush=True)

import requests
import csv
import time

# Base URL without the page parameter
base_url = 'https://danbooru.donmai.us/tags.json?limit=1000&search[hide_empty]=yes&search[is_deprecated]=no&search[order]=count'

# Specify the filename for the CSV
csv_filename = 'danbooru_tags_post_count_category_4.csv'

# Open a file to write
with open(csv_filename, mode='w', newline='', encoding='utf-8') as file:
    writer = csv.writer(file)
    
    # Write the header
    writer.writerow(['name', 'post_count'])

    # Loop through pages 1 to 1000
    for page in range(1, 1001):
        # Update the URL with the current page
        url = f'{base_url}&page={page}'
        
        # Fetch the JSON data
        response = requests.get(url)
        
        # Check if the request was successful
        if response.status_code == 200:
            data = response.json()
            
            # Break the loop if the data is empty (no more tags to fetch)
            if not data:
                print(f'No more data found at page {page}. Stopping.', flush=True)
                break
            
            # Write the data if category is 4
            for item in data:
                if item['category'] == 4:  # Only process tags where category is 4
                    writer.writerow([item['name'], item['post_count']])
            
            # Explicitly flush the data to the file
            file.flush()
        else:
            print(f'Failed to fetch data for page {page}. HTTP Status Code: {response.status_code}', flush=True)
            break

        print(f'Page {page} processed.', flush=True)
        # Sleep for 1 second so we don't DDOS Danbooru too much
        time.sleep(1)

print(f'Data has been written to {csv_filename}', flush=True)