commit dc0f8dccaa183afd54960fc31155d03364303a4d Author: Pablo Aramburo Date: Fri Sep 16 15:35:23 2022 -0600 [add] MVP diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4d7959f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +venv/ +proxies.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..89fad37 --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +# Proxy + +## Running the project +First create a list of live proxies +```bash +python3 scrape_proxies.py proxies.txt +``` + +Then run the mubeng instance through docker-compose +```bash +docker-compose up +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e44f5cf --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +version: '3.7' +services: + mubeng: + image: kitabisa/mubeng:v0.10.0 + container_name: mubeng + restart: always + ports: + - "8080:8080" + volumes: + - .:/mubeng + command: '--check --address 0.0.0.0:8080 --rotate 1 --method random --file /mubeng/proxies.txt --watch --timeout 10s' diff --git a/example.py b/example.py new file mode 100644 index 0000000..c583cab --- /dev/null +++ b/example.py @@ -0,0 +1,6 @@ +import requests + +proxies = {'https': 'http://localhost:8080', 'http': 'http://localhost:8080'} +ip = requests.get(url='http://canhasip.com/', proxies=proxies).text + +print(ip) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d15ce5a --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +requests==2.28.1 diff --git a/scrape_proxies.py b/scrape_proxies.py new file mode 100644 index 0000000..0938bc1 --- /dev/null +++ b/scrape_proxies.py @@ -0,0 +1,12 @@ +import requests +import json + +r = requests.get('https://raw.githubusercontent.com/fate0/proxylist/master/proxy.list') + +formated_proxies = [] +for line in r.text.splitlines(): + parsed = json.loads(line) + + if bool(parsed['anonymity'] == 'high_anonymous'): + proxy_string = f'{parsed["type"]}://{parsed["host"]}:{parsed["port"]}' + print(proxy_string) \ No newline at end of file