66 lines
2.1 KiB
Python
66 lines
2.1 KiB
Python
import os
|
|
import time
|
|
import urllib.parse
|
|
import requests
|
|
from requests.exceptions import RequestException
|
|
import concurrent.futures
|
|
|
|
def download_url(url):
|
|
proxies = {"http": "http://127.0.0.1:10808", "https": "http://127.0.0.1:10808"}
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
|
}
|
|
|
|
decoded_url = urllib.parse.unquote(url)
|
|
volume_name = decoded_url.split("/")[-1]
|
|
filename = f"html_files/{volume_name}.html"
|
|
|
|
if os.path.exists(filename):
|
|
# Already exists
|
|
return True, filename
|
|
|
|
success = False
|
|
retries = 3
|
|
while not success and retries > 0:
|
|
try:
|
|
response = requests.get(url, headers=headers, proxies=proxies, timeout=15)
|
|
if response.status_code == 200:
|
|
with open(filename, "w", encoding="utf-8") as out_f:
|
|
out_f.write(response.text)
|
|
success = True
|
|
print(f"Successfully downloaded {filename}")
|
|
return True, filename
|
|
else:
|
|
print(f"HTTP Error {response.status_code} for {url}. Retries left: {retries - 1}")
|
|
except RequestException as e:
|
|
pass
|
|
|
|
if not success:
|
|
retries -= 1
|
|
time.sleep(2)
|
|
|
|
if not success:
|
|
print(f"Failed to download {url} after all retries.")
|
|
return False, filename
|
|
|
|
return False, filename
|
|
|
|
def main():
|
|
if not os.path.exists("html_files"):
|
|
os.makedirs("html_files")
|
|
|
|
with open("missing_urls.txt", "r", encoding="utf-8") as f:
|
|
urls = [line.strip() for line in f if line.strip()]
|
|
|
|
# Use ThreadPoolExecutor to download multiple files concurrently
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
|
|
futures = [executor.submit(download_url, url) for url in urls]
|
|
for future in concurrent.futures.as_completed(futures):
|
|
try:
|
|
future.result()
|
|
except Exception as e:
|
|
print(f"Exception during download: {e}")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|