diff --git a/.gitignore b/.gitignore
index d2f8ff3..ec5358a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,3 @@
.git
-www/participants/**
+www/participants/*/*.csv
diff --git a/scripts/__pycache__/utils.cpython-311.pyc b/scripts/__pycache__/utils.cpython-311.pyc
index e1d2bbc..a3df33b 100644
Binary files a/scripts/__pycache__/utils.cpython-311.pyc and b/scripts/__pycache__/utils.cpython-311.pyc differ
diff --git a/scripts/lantern.py b/scripts/lantern.py
index c953b00..8cb3ced 100644
--- a/scripts/lantern.py
+++ b/scripts/lantern.py
@@ -1,42 +1,44 @@
-import os, pwd, re, pandas as pd, requests, shutil
-from PIL import Image
-import urllib
-import socks, socket, glob
from utils import *
+import os, pwd
+import pandas as pd
+import requests
+import shutil
+import time
+import urllib
+
+
def main():
- proxies = {
- 'http': 'socks5h://127.0.0.1:9050',
- 'https': 'socks5h://127.0.0.1:9050'
- }
+ os.system('clear')
+ proxies = {
+ 'http': 'socks5h://127.0.0.1:9050',
+ 'https': 'socks5h://127.0.0.1:9050'
+ }
- rootpath='/srv/darknet-lantern/'
- urlpath=pwd.getpwuid(os.getuid()).pw_dir+"/.darknet_participant_url"
- #print_colors(urlpath)
+ rootpath='/srv/darknet-lantern/'
+ urlpath=pwd.getpwuid(os.getuid()).pw_dir+"/.darknet_participant_url"
- # check if ~/.darknet_participant_url exists,
- # if exists, instance= the content of ~/.darknet_participant_url (which is the url: such as lantern.nowherejez...onion)
- print_colors("""
-;
- ED.
- E#Wi G: L. ,;
- E###G. j. E#, :EW: ,ft f#i
- E#fD#W; .. EW, E#t .GEE##; t#E .E#t GEEEEEEEL
- E#t t##L ;W, E##j E#t j#K;E###t t#E i#W, ,;;L#K;;.
- E#t .E#K, j##, E###D. E#GK#f E#fE#f t#E L#D. t#E
- E#t j##f G###, E#jG#W; E##D. E#t D#G t#E :K#Wfff; t#E
- E#t :E#K: :E####, E#t t##f E##Wi E#t f#E. t#E i##WLLLLt t#E
- E#t t##L ;W#DG##, E#t :K#E: E#jL#D: E#t t#K: t#E .E#L t#E
- E#t .D#W; j###DW##, E#KDDDD###iE#t ,K#jE#t ;#W,t#E f#E: t#E
- E#tiW#G. G##i,,G##, E#f,t#Wi,,,E#t jDE#t :K#D#E ,WW; t#E
- E#K##i :K#K: L##, E#t ;#W: j#t E#t .E##E .D#; t#E
- E##D. ;##D. L##, DWi ,KK: ,; .. G#E tt fE
- E#t ,,, .,, fE :
- L: ,
-
-
- L. ,; L.
+ print_colors("""
+;
+ ED.
+ E#Wi G: L. ,;
+ E###G. j. E#, :EW: ,ft f#i
+ E#fD#W; .. EW, E#t .GEE##; t#E .E#t GEEEEEEEL
+ E#t t##L ;W, E##j E#t j#K;E###t t#E i#W, ,;;L#K;;.
+ E#t .E#K, j##, E###D. E#GK#f E#fE#f t#E L#D. t#E
+ E#t j##f G###, E#jG#W; E##D. E#t D#G t#E :K#Wfff; t#E
+ E#t :E#K: :E####, E#t t##f E##Wi E#t f#E. t#E i##WLLLLt t#E
+ E#t t##L ;W#DG##, E#t :K#E: E#jL#D: E#t t#K: t#E .E#L t#E
+ E#t .D#W; j###DW##, E#KDDDD###iE#t ,K#jE#t ;#W,t#E f#E: t#E
+ E#tiW#G. G##i,,G##, E#f,t#Wi,,,E#t jDE#t :K#D#E ,WW; t#E
+ E#K##i :K#K: L##, E#t ;#W: j#t E#t .E##E .D#; t#E
+ E##D. ;##D. L##, DWi ,KK: ,; .. G#E tt fE
+ E#t ,,, .,, fE :
+ L: ,
+
+
+ L. ,; L.
i EW: ,ft f#i j. EW: ,ft
LE .. E##; t#E GEEEEEEEL .E#t EW, E##; t#E
L#E ;W, E###t t#E ,;;L#K;;. i#W, E##j E###t t#E
@@ -50,86 +52,77 @@ def main():
:W##########Wt ;##D. L##, .. G#E fE tt DWi ,KK: .. G#E
:,,,,,,,,,,,,,.,,, .,, fE : fE
, ,
- """, bold=True)
- isitvalid="n"
- while isitvalid != "y":
- if os.path.isfile(urlpath):
- with open(urlpath) as f:
- instance = f.read().rstrip()
- # check if the instance URL domain is valid
- #print_colors(urlpath,instance)
- if IsOnionValid(instance):
- print_colors(f'[+] Instance Name: {instance} {IsOnionValid(instance)}')
- isitvalid="y"
- else:
- print_colors(f'[-] Invalid instance name in ~/.darknet_participant_url: {instance}',is_error=True )
- return False
- else:
- print_colors("[+] Instance Path doesn't exist yet")
- # and ask for the instance URL domain
- instance = input("What is your Instance domain ? (ex: lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion): ")
- instancepath=rootpath+'www/participants/'+instance
- # check if the instance URL domain is valid
- if IsOnionValid(instance):
- print_colors(f'[+] Instance Name: {instance} {IsUrlValid(instance)}')
- else:
- print_colors(f'[-] Invalid instance name in ~/.darknet_participant_url: {instance}',is_error=True )
- return False
+ version: 1.0.1
+ """, bold=True)
-
- # ask the user if the instance URL is valid ?
- print_colors()
- print_colors(instance)
- isitvalid=input("Is this your this your instance domain ? (y/n) ")
- # if yes, then write it into ~/.darknet_participant_url
- if isitvalid == "y" :
- print_colors("OK writing the instance url to ~/.darknet_participants_url")
- with open(urlpath, "w") as file:
- file.write(instance)
- print_colors("[+] file written, let's read it")
- f = open(urlpath,"r")
- print_colors(f.read())
- print_colors("[+] Initial Setup Completed!",highlight=True)
- myinstance = instance
- instancepath=rootpath+'www/participants/'+instance
- templatepath=rootpath+'templates/'
- verifiedcsvfile=instancepath+'/verified.csv'
- unverifiedcsvfile=instancepath+'/unverified.csv'
- blcsvfile=instancepath+'/blacklist.csv'
- secsvfile=instancepath+'/sensitive.csv'
- webpcsvfile=instancepath+'/webring-participants.csv'
- # check if instancepath exists, if not then create the directory
- if not os.path.exists(instancepath):
- os.makedirs(instancepath)
- # check if all the required csv files exist in it, otherwise copy them from the templates directory
- # NOTE : the templates files are EMPTY by default, this is because i want each peer to manually review lists of links, and links themselves manually, this is to avoid allowing malicious links to slip through without intentional edits from the peer themselves.
- for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv','banner.png']:
- filepath=instancepath+'/'+i
- if not os.path.isfile(filepath):
- # copy templates/ FILE.CSV to instancepath/ FILE.CSV
- src=templatepath+i
- shutil.copyfile(src, filepath)
- # now that they exist, get vdf and uvdf and the rest
- vdf = pd.read_csv(verifiedcsvfile)
- uvdf = pd.read_csv(unverifiedcsvfile)
- bldf = pd.read_csv(blcsvfile)
- sedf = pd.read_csv(secsvfile)
- webpdf = pd.read_csv(webpcsvfile)
- print_colors("[+] file exists, your Webring URL is ", instance)
- isitvalid = "y"
+ while True:
+ if os.path.isfile(urlpath):
+ with open(urlpath) as f:
+ instance = f.read().rstrip()
+ if IsOnionValid(instance):
+ print_colors(f"[+] Instance Name: {instance}. Valid:{IsOnionValid(instance)}")
+ break
+ else:
+ print_colors(f'[-] Invalid instance name in ~/.darknet_participant_url: {instance}',is_error=True )
+ break
+ else:
+ print_colors("[+] Instance Path doesn't exist yet")
+ print_colors(f"Your url will be saved here {urlpath}")
+ instance = input("What is your Instance domain?(ex: lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion): ")
+ if IsOnionValid(instance):
+ print_colors(f"[+] Instance Name: {instance}. Valid: {IsUrlValid(instance)}")
+ instancepath=rootpath+'www/participants/'+instance
+ else:
+ print_colors(f'[-] Invalid instance name in ~/.darknet_participant_url: {instance}', is_error=True )
+ break
- while True:
- print_colors("""
+ isitvalid=input("Is this your this your instance domain?(y/n) ")
+ if isitvalid == "y" :
+ print_colors("OK writing the instance url to ~/.darknet_participants_url")
+ with open(urlpath, "w") as file:
+ file.write(instance)
+ print_colors("[+] File written")
+ f = open(urlpath,"r")
+ print_colors(f"{f.read()}")
+ print_colors("[+] Initial Setup Completed!")
+ myinstance = instance
+ instancepath=rootpath+'www/participants/'+instance
+ templatepath=rootpath+'templates/'
+ verifiedcsvfile=instancepath+'/verified.csv'
+ unverifiedcsvfile=instancepath+'/unverified.csv'
+ blcsvfile=instancepath+'/blacklist.csv'
+ secsvfile=instancepath+'/sensitive.csv'
+ webpcsvfile=instancepath+'/webring-participants.csv'
+ if not os.path.exists(instancepath):
+ print_colors(f"{rootpath}",is_error=True, bold=True)
+ os.makedirs(instancepath)
+ # check if all the required csv files exist in it, otherwise copy them from the templates directory
+ # NOTE : the templates files are EMPTY by default, this is because i want each peer to manually review lists of links, and add links themselves manually, this is to avoid allowing malicious links to slip through without intentional edits from the peer themselves.
+ for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv','banner.png']:
+ filepath=instancepath+'/'+i
+ if not os.path.isfile(filepath):
+ # copy templates/ FILE.CSV to instancepath/ FILE.CSV
+ src=templatepath+i
+ shutil.copyfile(src, filepath)
+ # now that they exist, get vdf and uvdf and the rest
+ vdf = pd.read_csv(verifiedcsvfile)
+ uvdf = pd.read_csv(unverifiedcsvfile)
+ bldf = pd.read_csv(blcsvfile)
+ sedf = pd.read_csv(secsvfile)
+ webpdf = pd.read_csv(webpcsvfile)
+ print_colors(f"[+] file exists, your Webring URL is {instance}")
+
+ while True:
+ time.sleep(1.5)
+ print_colors("""
[+] Welcome to your own Darknet Lantern Instance, where you can explore the Darknet and help others do the same.
-""",highlight=True)
- print_colors("""
Managing Websites:
- 1) Add a new Website entry (into unverified.csv)
- 2) Trust a Website entry (move an entry from unverified to verified.csv)
- 3) Untrust a Website entry (move an entry from unverified to verified.csv)
+ 1) Add a new Website entry (into unverified.csv)
+ 2) Trust a Website entry (move an entry from unverified to verified.csv)
+ 3) Untrust a Website entry (move an entry from unverified to verified.csv)
Managing Webring Participants:
4) Synchronize new links from existing webring participants, into your unverified.csv file
@@ -142,148 +135,143 @@ Managing Wordlists:
Maintenance:
9) Remove the duplicate URLs for your own instance
- 10) Perform sanity checks on all csv files for all instances (to mark them as sensitive / or remove the ones that are blacklisted)
+ 10) Perform sanity checks on all csv files for all instances (to mark them as sensitive / or remove the ones that are blacklisted)
0) Exit
- """)
- option = input("Select Option? (0-11): ")
- print_colors(option)
- match option:
+ """)
+ option = input("Select an option? (0-10): ").strip()
+ try:
+ option = int(option)
+ except ValueError:
+ print_colors(f"[-] Exiting. {option} is not a valid option.", bold=True, is_error=True)
+ break
+ match option:
+
+ ########## MANAGING WEBSITE ENTRIES #################
+ #Websites:
+ # 1) Add a new Website entry (into unverified.csv)
+ # 2) Trust a Website entry (move an entry from unverified to verified.csv)
+ # 3) Untrust a Website entry (move an entry from unverified to verified.csv)
+ #####################################################
+ case 1:
+ while True:
+ print_colors("\n[+] Add a new Website entry (into unverified.csv)")
+ name=''
+ while(IsNameValid(name) is not True):
+ name = input("What is the name of the website? ")
+ category=''
+ while(IsCategoryValid(category) is not True):
+ category = input("What is the website Category? ")
+ # the url of the website (required) + check if its valid
+ url=''
+ while(IsUrlValid(url) is not True):
+ url=input("What is the website URL ? ")
+
+ # a quick description (optional) + check if its valid
+ desc='DEFAULT'
+ while(IsDescriptionValid(desc) is not True):
+ desc=input("Description for the website ? (Optional) ")
+ choice=input("Is the website sensitive ? (ex: related to drugs) (y/n) ")
+ if choice == "n":
+ sensi = 'NO'
+ else:
+ sensi = 'YES'
+
+ newrow=[instance,category,name,url,sensi,desc,'YES','100']
+ print_colors(f"[+] NEWROW= {newrow}")
+ # (rest is automatic: status, score, instance is = '' because it is your own instance)
+ # TODO check if the entry doesn't already exist in verified.csv and in unverified.csv
+ # if it doesnt exist, add it into unverified.csv
+ uvdf.loc[-1] = newrow # adding a row
+ uvdf.index = uvdf.index + 1 # shifting index
+ uvdf = uvdf.sort_index() # sorting by index
+ uvdf = uvdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
+ print_colors("[+] New row added! now writing the csv file")
+ uvdf.to_csv(unverifiedcsvfile, index=False)
+ choice=input("\n[+] Want to add another website ? (y/n) ")
+ if choice == "n":
+ break
-########## MANAGING WEBSITE ENTRIES #################
-#Websites:
-# 1) Add a new Website entry (into unverified.csv)
-# 2) Trust a Website entry (move an entry from unverified to verified.csv)
-# 3) Untrust a Website entry (move an entry from unverified to verified.csv)
-#####################################################
+ case 2:
+ print_colors("[+] Trust a Website entry (move an entry from unverified to verified.csv)")
+ while True:
+ vdf = pd.read_csv(verifiedcsvfile)
+ uvdf = pd.read_csv(unverifiedcsvfile)
+ # search for a word
+ print_colors(f"{uvdf[['Name','URL']]}")
+ name=''
+ while(IsNameValid(name) is not True):
+ name = input("What is the Website name you want to trust ? (ex: Nowhere)")
+ filter_uvdf = uvdf[uvdf.Name.str.contains(name)]
+ # NOTE and display only the matching entries in unverified.csv in an array format (display it in CLI).
+ print_colors(f"{filter_uvdf[['Name','URL']]}")
+ # check if there are no results, dont proceed if there are none!
+ if filter_uvdf.size == 0:
+ print_colors("ERROR no results, skipping.",is_error=True)
+ else:
+ # Each of the rows has an index,
+ index=-1
+ while (index not in filter_uvdf.index):
+ index = int(input("What is the index of the entry that you want to move to verified.csv ? (ex: 3) "))
+ # once selected, it must be able to SAVE and print_colors that row:
+ print_colors(f"{uvdf.iloc[index].values}")
+ newrow=uvdf.iloc[index].values
- case "1":
- done = False
- while done == False:
- print_colors("\n[+] Add a new Website entry (into unverified.csv)")
- name=''
- while(IsNameValid(name) is not True):
- name = input("What is the Website name ? ")
- category=''
- while(IsCategoryValid(category) is not True):
- category = input("What is the website Category ? ")
- # the url of the website (required) + check if its valid
- url=''
- while(IsUrlValid(url) is not True):
- url=input("What is the website URL ? ")
- # a quick description (optional) + check if its valid
- desc='DEFAULT'
- while(IsDescriptionValid(desc) is not True):
- desc=input("Description for the website ? (Optional) ")
- # sensitive ? (y/n) + check if its valid
- #entry_sensi = input("is it a sensitive website ? (ex: website related to drugs) (y/n)")
+ # append it into verified.csv
+ vdf.loc[-1] = newrow # adding a row
+ vdf.index = vdf.index + 1 # shifting index
+ vdf = vdf.sort_index() # sxorting by index
+ vdf = vdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
+ vdf.to_csv(verifiedcsvfile, index=False)
+ print_colors("[+] New row added to verified.csv! now writing to the csv")
- choice=input("Is the website sensitive ? (ex: related to drugs) (y/n) ")
- if choice == "n":
- sensi = '❌'
- else:
- sensi = '✔️'
- newrow=[instance,category,name,url,sensi,desc,'','']
- print_colors("[+] NEWROW=",newrow)
- # (rest is automatic: status, score, instance is = '' because it is your own instance)
- # TODO check if the entry doesn't already exist in verified.csv and in unverified.csv
- # if it doesnt exist, add it into unverified.csv
- uvdf.loc[-1] = newrow # adding a row
- uvdf.index = uvdf.index + 1 # shifting index
- uvdf = uvdf.sort_index() # sorting by index
- uvdf = uvdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
- print_colors("[+] New row added! now writing the csv file:")
- uvdf.to_csv(unverifiedcsvfile, index=False)
- choice=input("\n[+] Want to add another website ? (y/n) ")
- if choice == "n":
- done = True
-
+ # remove it from unverified.csv
+ uvdf.drop(index, inplace= True)
+ uvdf = uvdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
+ uvdf.to_csv(unverifiedcsvfile, index=False)
+ print_colors("[+] Link is now moved to verified.csv!")
+ choice=input("\n[+] Want to trust another website ? (y/n) ")
+ if choice == "n":
+ break
- case "2":
- print_colors("[+] Trust a Website entry (move an entry from unverified to verified.csv)")
- done = False
- while done == False:
- vdf = pd.read_csv(verifiedcsvfile)
- uvdf = pd.read_csv(unverifiedcsvfile)
- # search for a word
- print_colors(uvdf[['Name','URL']])
- name=''
- while(IsNameValid(name) is not True):
- name = input("What is the Website name you want to trust ? (ex: Nowhere)")
- filter_uvdf = uvdf[uvdf.Name.str.contains(name)]
- # NOTE and display only the matching entries in unverified.csv in an array format (display it in CLI).
- print_colors(filter_uvdf[['Name','URL']])
- # check if there are no results, dont proceed if there are none!
- if filter_uvdf.size == 0:
- print_colors("ERROR no results, skipping.")
- else:
- # Each of the rows has an index,
- index=-1
- while (index not in filter_uvdf.index):
- # prompt the user to ask for with row they want to move to verified.csv
- index = int(input("What is the index of the entry that you want to move to verified.csv ? (ex: 3) "))
- # once selected, it must be able to SAVE and print_colors that row:
- print_colors(uvdf.iloc[index].values)
- newrow=uvdf.iloc[index].values
-
+ case 3:
+ print_colors("[+] Untrust a Website entry (move an entry from verified to unverified.csv)")
+ print_colors(f"{vdf[['Name','URL']]}")
+ # search for a word
+ name=''
+ while(IsNameValid(name) is not True):
+ name = input("What is the Website name you want to untrust? (ex: BreachForums)")
+ filter_vdf = vdf[vdf.Name.str.contains(name)]
+ # and display only the matching entries in unverified.csv in an array format (display it in CLI).
+ print_colors(f"{filter_vdf[['Name','URL']]}")
+ # check if there are no results, dont proceed if there are none!
+ if filter_vdf.size == 0:
+ print_colors("ERROR no results, skipping.", is_error=True)
+ else:
+ # Each of the rows has an index,
+ index=-1
+ while (index not in filter_vdf.index):
+ index = int(input("What is the index of the entry that you want to move to unverified.csv ? (ex: 3) "))
+ # once selected, it must be able to SAVE and print_colors that row:
+ print_colors(vdf.iloc[index].values)
+ newrow=vdf.iloc[index].values
- # append it into verified.csv
- vdf.loc[-1] = newrow # adding a row
- vdf.index = vdf.index + 1 # shifting index
- vdf = vdf.sort_index() # sxorting by index
- vdf = vdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
- vdf.to_csv(verifiedcsvfile, index=False)
- print_colors("[+] New row added to verified.csv! now writing to the csv")
-
- # remove it from unverified.csv
- uvdf.drop(index, inplace= True)
- uvdf = uvdf.sort_values(by=["Category","Score"], ascending=[True,False]) # sorting categories
- uvdf.to_csv(unverifiedcsvfile, index=False)
- print_colors("[+] Link is now moved to verified.csv!")
- choice=input("\n[+] Want to trust another website ? (y/n) ")
- if choice == "n":
- done = True
+ # append it into unverified.csv
+ uvdf.loc[-1] = newrow # adding a row
+ uvdf.index = uvdf.index + 1 # shifting index
+ uvdf = uvdf.sort_index() # sorting by index
+ uvdf.to_csv(unverifiedcsvfile, index=False)
+ print_colors("[+] New row added to unverified.csv!")
- case "3":
- print_colors("[+] Untrust a Website entry (move an entry from verified to unverified.csv)")
- print_colors(vdf[['Name','URL']])
- # search for a word
- name=''
- while(IsNameValid(name) is not True):
- name = input("What is the Website name you want to untrust ? (ex: BreachForums)")
- filter_vdf = vdf[vdf.Name.str.contains(name)]
- # and display only the matching entries in unverified.csv in an array format (display it in CLI).
- print_colors(filter_vdf[['Name','URL']])
- # check if there are no results, dont proceed if there are none!
- if filter_vdf.size == 0:
- print_colors("ERROR no results, skipping.")
- else:
- # Each of the rows has an index,
- index=-1
- while (index not in filter_vdf.index):
- # prompt the user to ask for with row they want to move to unverified.csv
- index = int(input("What is the index of the entry that you want to move to unverified.csv ? (ex: 3) "))
- # once selected, it must be able to SAVE and print_colors that row:
- print_colors(vdf.iloc[index].values)
- newrow=vdf.iloc[index].values
-
- # append it into unverified.csv
- uvdf.loc[-1] = newrow # adding a row
- uvdf.index = uvdf.index + 1 # shifting index
- uvdf = uvdf.sort_index() # sorting by index
- uvdf.to_csv(unverifiedcsvfile, index=False)
- print_colors("[+] New row added to unverified.csv!")
-
-
- # remove it from verified.csv
- vdf.drop(index, inplace= True)
- vdf.to_csv(verifiedcsvfile, index=False)
- print_colors("[+] Link is now moved to unverified.csv!")
+ # remove it from verified.csv
+ vdf.drop(index, inplace= True)
+ vdf.to_csv(verifiedcsvfile, index=False)
+ print_colors("[+] Link is now moved to unverified.csv!")
####### MANAGING WEBRING PARTICIPANTS ###########
@@ -294,191 +282,170 @@ Maintenance:
+ #check if it works when you have a second webring participant
+ case 4:
+ print_colors("4) Synchronize new links from existing webring participants, into your unverified.csv file")
+ participantsdir=rootpath+'www/participants/'
+ name=''
+ desc=''
+ trusted=''
+ status=''
+ score=''
+ webringcsvfile=instancepath+'/'+'webring-participants.csv'
+ wdf = pd.read_csv(webringcsvfile)
+ for participant in os.listdir(participantsdir):
+ participantdir=participantsdir+participant
+
+ # NOTE check if the webring participant is yourself, if it is, then skip it
+ if participant != myinstance: # prod: dont use your own intance
+ #if participant == myinstance: # preprod testing only on your own instance
+ #overwrite the existing files in the participant's directory, with their version (download all the csv files from them again)
+ basewurl='http://'+participant+'/participants/'+participant+'/'
+ print_colors(f"{basewurl}")
+ print_colors(f"[+] Downloading the files of: {participant} ")
+ w_vcsv=basewurl+'verified.csv'
+ w_uvcsv=basewurl+'unverified.csv'
+ w_blcsv=basewurl+'blacklist.csv'
+ w_scsv=basewurl+'sensitive.csv'
+ w_webcsv=basewurl+'webring-participants.csv'
+
+ # verify that their verified.csv csv file exists at basewurl+'verified.csv'
+ if CheckUrl(w_vcsv) is False or CheckUrl(w_uvcsv) is False or CheckUrl(w_blcsv) is False or CheckUrl(w_scsv) is False or CheckUrl(w_webcsv) is False:
+ print_colors("[-] Webring Participant isn't reachable, skipping", is_error=True)
+ else: #if the webring participant is reachable, proceed
+ print_colors("[+] Webring Participant is reachable, updating their csv files:")
+ for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv']:
+ # FOR EACH CSV FILE TO GET:
+ # URL: basewurl / FILE.CSV
+ # PATH: participantdir / FILE.CSV
+ # download the external csv file and save it into the "text" variable:
+ #response = urllib.request.urlopen(basewurl+i)
+ response = requests.get(basewurl+i, proxies=proxies)
+ #data = response.read() # a `bytes` object
+ #text = data.decode('utf-8')
+ text = response.text
+ # save the text variable into the destination file:
+ csvfilepath=participantdir+'/'+i
+ with open(csvfilepath, "w") as file:
+ file.write(text)
+ f = open(csvfilepath,"r")
+
+ # download the banner.png image:
+
+ bannerurl=basewurl+'banner.png'
+ bannerpath=participantdir+'/banner.png'
+ r = requests.get(bannerurl, stream=True, proxies=proxies)
+ with open(bannerpath, 'wb') as f:
+ r.raw.decode_content = True
+ shutil.copyfileobj(r.raw, f)
+
+ # SANITY CHECK ON THE BANNER PNG IMAGE:
+ if IsBannerValid(bannerpath):
+ pass
+ else:
+ # if false, overwrite it with the template banner png file
+ os.remove(bannerpath)
+ # copy templates/banner.png to bannerpath
+ bannertemplatepath=templatepath+'banner.png'
+ shutil.copyfile(bannertemplatepath, bannerpath)
- #check if it works when you have a second webring participant
- case "4":
- print_colors("4) Synchronize new links from existing webring participants, into your unverified.csv file")
- # iterate through each existing directories in www/participants/* to get each webring participant
- participantsdir=rootpath+'www/participants/'
- #print_colors(os.listdir(participantsdir))
- name=''
- desc=''
- trusted=''
- status=''
- score=''
- webringcsvfile=instancepath+'/'+'webring-participants.csv'
- wdf = pd.read_csv(webringcsvfile)
- for participant in os.listdir(participantsdir):
- participantdir=participantsdir+participant
- #print_colors(participant)
-
- # NOTE check if the webring participant is yourself, if it is, then skip it
- if participant != myinstance: # prod: dont use your own intance
- #if participant == myinstance: # preprod testing only on your own instance
- #overwrite the existing files in the participant's directory, with their version (download all the csv files from them again)
- basewurl='http://'+participant+'/participants/'+participant+'/'
- print_colors(basewurl)
- print_colors('[+] Downloading the files of ',participant, ": ")
- w_vcsv=basewurl+'verified.csv'
- w_uvcsv=basewurl+'unverified.csv'
- #print_colors(CheckUrl(w_uvcsv))
- w_blcsv=basewurl+'blacklist.csv'
- #print_colors(CheckUrl(w_blcsv))
- w_scsv=basewurl+'sensitive.csv'
- #print_colors(CheckUrl(w_scsv))
- w_webcsv=basewurl+'webring-participants.csv'
- #print_colors(CheckUrl(w_webcsv))
+ # check if the participant is already listed in webring-participants.csv or not, and add them if not already listed
+ # and display only the matching entries in unverified.csv in an array format (display it in CLI).
+ filter_wdf = wdf[wdf.URL.str.contains(participant)]
+ # check if there are no results, dont proceed if there are none!
+ if filter_wdf.size == 0: #skip if webring participant is already listed, otherwise proceed
+ newrow=[name,participant,desc,trusted,status,score]
+ wdf.loc[-1] = newrow # adding a row
+ wdf.index = wdf.index + 1 # shifting index
+ wdf = wdf.sort_index() # sorting by index
+ wdf.to_csv(webringcsvfile, index=False)
+ else:
+ pass
- # verify that their verified.csv csv file exists at basewurl+'verified.csv'
- if CheckUrl(w_vcsv) is False or CheckUrl(w_uvcsv) is False or CheckUrl(w_blcsv) is False or CheckUrl(w_scsv) is False or CheckUrl(w_webcsv) is False:
- print_colors("[-] Webring Participant isn't reachable, skipping")
- #return False #dont do anything if the webring participant isnt reachable.
- else: #if the webring participant is reachable, proceed
- print_colors("[+] Webring Participant is reachable, updating their csv files:")
- for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv']:
- # FOR EACH CSV FILE TO GET:
- # URL: basewurl / FILE.CSV
- # PATH: participantdir / FILE.CSV
- #print_colors('[+] DOWNLOADING ',basewurl+i)
- # download the external csv file and save it into the "text" variable:
- #response = urllib.request.urlopen(basewurl+i)
- response = requests.get(basewurl+i, proxies=proxies)
- #data = response.read() # a `bytes` object
- #text = data.decode('utf-8')
- text = response.text
- # save the text variable into the destination file:
- #print_colors('[+] SAVING IT INTO ',participantdir+'/'+i)
- csvfilepath=participantdir+'/'+i
- with open(csvfilepath, "w") as file:
- file.write(text)
- #print_colors("[+] file written, let's read it")
- f = open(csvfilepath,"r")
- #print_colors(f.read())
-
- # download the banner.png image:
-
- bannerurl=basewurl+'banner.png'
- bannerpath=participantdir+'/banner.png'
- r = requests.get(bannerurl, stream=True, proxies=proxies)
- with open(bannerpath, 'wb') as f:
- r.raw.decode_content = True
- shutil.copyfileobj(r.raw, f)
-
- # SANITY CHECK ON THE BANNER PNG IMAGE:
- if IsBannerValid(bannerpath):
- print_colors('[+] Banner is valid')
- pass
- else:
- # if false, overwrite it with the template banner png file
- print_colors('[-] Banner is not valid, replacing it with the default banner')
- os.remove(bannerpath)
- # copy templates/banner.png to bannerpath
- bannertemplatepath=templatepath+'banner.png'
- shutil.copyfile(bannertemplatepath, bannerpath)
-
-
- #print_colors("[+] Webring Participant is valid, adding it if it's not already added.")
- #print_colors('[+] PARTICIPANT=',participant)
- # check if the participant is already listed in webring-participants.csv or not, and add them if not already listed
- # and display only the matching entries in unverified.csv in an array format (display it in CLI).
- filter_wdf = wdf[wdf.URL.str.contains(participant)]
- #print_colors(filter_wdf[['Name','URL']])
- # check if there are no results, dont proceed if there are none!
- if filter_wdf.size == 0: #skip if webring participant is already listed, otherwise proceed
- newrow=[name,participant,desc,trusted,status,score]
- #print_colors("[+] NEWROW=",newrow)
- wdf.loc[-1] = newrow # adding a row
- wdf.index = wdf.index + 1 # shifting index
- wdf = wdf.sort_index() # sorting by index
- #print_colors("[+] New row added! now writing the csv file:",webringcsvfile)
- wdf.to_csv(webringcsvfile, index=False)
- else:
- pass
- #print_colors('[+] Webring participant is already listed in your own webring-participants.csv file!')
-
- # iterate through the participant's verified.csv and unverified.csv files
- for w in ['verified.csv','unverified.csv']:
- csvfilepath=participantdir+'/'+w
- print_colors(csvfilepath)
- csvdf = pd.read_csv(csvfilepath)
- #print_colors(bldf[['blacklisted-words']])
- bldf[['blacklisted-words']].iterrows()
- rows2delete= [] # it is an empty list at first
- for i,j in csvdf.iterrows():
- #print_colors("[+] Unverified.csv ROW=",i, uvdf.at[i, 'Instance'], uvdf.at[i, 'Category'], uvdf.at[i, 'Name'], uvdf.at[i, 'URL'], uvdf.at[i, 'Description'])
- #print_colors("[+] Unverified.csv ROW=",i, uvdf.iloc[[i]])
- #row=uvdf.iloc[[i]] #it displays the index
- row=csvdf.loc[i,:].values.tolist()
- print_colors(row)
- #print_colors(i,row)
+ # iterate through the participant's verified.csv and unverified.csv files
+ for w in ['verified.csv','unverified.csv']:
+ csvfilepath=participantdir+'/'+w
+ print_colors(f"{csvfilepath}")
+ csvdf = pd.read_csv(csvfilepath)
+ bldf[['blacklisted-words']].iterrows()
+ rows2delete= [] # it is an empty list at first
+ for i,j in csvdf.iterrows():
+ row=csvdf.loc[i,:].values.tolist()
+ print_colors(f"{row}")
- ################################ SANITY CHECKS ####################################
- ### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion###
- #print_colors("[+] ROW=",i,"ROW CONTENTS=", IsUrlValid(uvdf.at[i, 'Instance']), IsCategoryValid(uvdf.at[i, 'Category']), IsNameValid(uvdf.at[i, 'Name']), IsUrlValid(uvdf.at[i, 'URL']), IsStatusValid(uvdf.at[i, 'Sensitive']), IsDescriptionValid(uvdf.at[i, 'Description']), IsStatusValid(uvdf.at[i, 'Status']), IsScoreValid(uvdf.at[i, 'Score']))
- if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False:
- #mark the row for deletion as it has invalid inputs
- if i not in rows2delete:
- print_colors("Marking row", i,"for deletion, as it has invalid inputs")
- rows2delete.append(i) #mark the row for deletion if not already done
+ ################################ SANITY CHECKS ####################################
+ ### SANITY CHECK 0: make sure that ✔️ and x are replaced with YES/NO, as it changed since v1.0.1 ###
+ if csvdf.at[i, 'Status'] == "✔️" or csvdf.at[i, 'Status'] == "YES" :
+ csvdf.at[i, 'Status'] = "YES"
+ csvdf.to_csv(csvfilepath, index=False)
+ else:
+ csvdf.at[i, 'Status'] = "NO"
+ csvdf.to_csv(csvfilepath, index=False)
+
+ if csvdf.at[i, 'Sensitive'] == "✔️" or csvdf.at[i, 'Sensitive'] == "YES" :
+ csvdf.at[i, 'Sensitive'] = "YES"
+ csvdf.to_csv(csvfilepath, index=False)
+ else:
+ csvdf.at[i, 'Sensitive'] = "NO"
+ csvdf.to_csv(csvfilepath, index=False)
- ### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ###
- for k,l in bldf.iterrows():
- #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words'])
- blword=bldf.at[k, 'blacklisted-words']
- if any(blword in str(x) for x in row) == True:
- #print_colors("found blacklisted word! marking row for deletion")
- if i not in rows2delete:
- print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word")
- rows2delete.append(i) #mark the row for deletion if not already done
- else:
- if i not in rows2delete:
- # not a blacklisted link, therefore it is suitable to be added to your own csv files:
- ################################ CHECKING FOR DUPLICATES! #########################
- # for each link in the participant's verified/unverified csv files,
- # check if the link is already listed in your own verified.csv or unverified.csv
- filterterm=csvdf.at[i, 'URL']
- filter_vdf= vdf[vdf.URL.str.contains(filterterm)]
- filter_uvdf= uvdf[uvdf.URL.str.contains(filterterm)]
- if len(filter_uvdf.index) == 0 and len(filter_vdf.index) == 0:
- #if link doesnt exist in either of your verified/unverified csv files,
- # then add it to your own unverified.csv file:
- newrow=row
- uvdf.loc[-1] = newrow # adding a row
- uvdf.index = uvdf.index + 1 # shifting index
- uvdf = uvdf.sort_index() # sorting by index
- uvdf.to_csv(unverifiedcsvfile, index=False)
- print_colors("[+] New row added to your own unverified.csv file!")
- else:
- #print_colors('[-] Skipping row as it is already added in', w, row, is_error=True )
- print_colors(f'[-] Skipping row as it is already added in {w}, {row}', is_error=True )
+ ### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion###
+ if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False:
+ #mark the row for deletion as it has invalid inputs
+ if i not in rows2delete:
+ print_colors(f"Marking row {i} for deletion, as it has invalid inputs")
+ rows2delete.append(i) #mark the row for deletion if not already done
-
-
- ###################### APPENDING TO YOUR OWN UNVERIFIED.CSV FILE###################
+ ### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ###
+ for k,l in bldf.iterrows():
+ blword=bldf.at[k, 'blacklisted-words']
+ if any(blword in str(x) for x in row) == True:
+ if i not in rows2delete:
+ print_colors(f"Marking row {i} for deletion, as it matches with a blacklisted word")
+ rows2delete.append(i) #mark the row for deletion if not already done
+ else:
+ if i not in rows2delete:
+ # not a blacklisted link, therefore it is suitable to be added to your own csv files:
+ ################################ CHECKING FOR DUPLICATES! #########################
+ # for each link in the participant's verified/unverified csv files,
+ # check if the link is already listed in your own verified.csv or unverified.csv
+ filterterm=csvdf.at[i, 'URL']
+ filter_vdf= vdf[vdf.URL.str.contains(filterterm)]
+ filter_uvdf= uvdf[uvdf.URL.str.contains(filterterm)]
+ if len(filter_uvdf.index) == 0 and len(filter_vdf.index) == 0:
+ newrow=row
+ uvdf.loc[-1] = newrow # adding a row
+ uvdf.index = uvdf.index + 1 # shifting index
+ uvdf = uvdf.sort_index() # sorting by index
+ uvdf.to_csv(unverifiedcsvfile, index=False)
+ print_colors("[+] New row added to your own unverified.csv file!")
+ else:
+ print_colors('[-] Skipping row as it is already added in {w} {row}',is_error=True)
- ### SANITY CHECK 3: Mark all the rows that are supposed to be sensitive ###
- for k,l in sedf.iterrows():
- #print_colors("[+] Sensitive word=",k, sedf.at[k, 'sensitive-words'])
- seword=sedf.at[k, 'sensitive-words']
- if any(seword in str(x) for x in row) == True:
- if csvdf.at[i, 'Sensitive'] != '✔️':
- print_colors("Marking row", i,"as sensitive, as it matches with a sensitive word")
- csvdf.at[i, 'Sensitive']='✔️'
- print_colors(f'[-] Rows to delete: {rows2delete}')
- # only delete rows after you've gone through all the unverified.csv OR verified.csv rows'
- for i in rows2delete:
- row=csvdf.loc[i,:].values.tolist()
- print_colors('[+] REMOVING ROW :',i,row)
- csvdf.drop(i, inplace= True)
- csvdf.to_csv(csvfilepath, index=False)
- rows2delete= [] # it is an empty list at first
+ ###################### APPENDING TO YOUR OWN UNVERIFIED.CSV FILE###################
-
+
+ ### SANITY CHECK 3: Mark all the rows that are supposed to be sensitive ###
+ for k,l in sedf.iterrows():
+ seword=sedf.at[k, 'sensitive-words']
+ if any(seword in str(x) for x in row) == True:
+ if csvdf.at[i, 'Sensitive'] != 'NO':
+ print_colors("Marking row", i,"as sensitive, as it matches with a sensitive word")
+ csvdf.at[i, 'Sensitive']='YES'
+
+ print_colors(f'[-] Rows to delete: {rows2delete}', is_error=True)
+ # only delete rows after you've gone through all the unverified.csv OR verified.csv rows'
+ for i in rows2delete:
+ row=csvdf.loc[i,:].values.tolist()
+ print_colors(f'[+] REMOVING ROW: {i}{row}')
+ csvdf.drop(i, inplace= True)
+ csvdf.to_csv(csvfilepath, index=False)
+ rows2delete= [] # it is an empty list at first
@@ -486,156 +453,144 @@ Maintenance:
- case "5":
- print_colors("[+] Add a new webring participant (and download their files into their directory (without trusting them yet!))")
- webring_participant_url = ''
- while(IsOnionValid(webring_participant_url) is not True):
- # ask for the url to the other webring participant and check if the (onion only) url is valid or not:
- webring_participant_url = input("What is the onion domain of the new webring participant? (ex: lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion) ")
- # check if the directory exists locally or not,
- participantdir=rootpath+'www/participants/'+webring_participant_url
- #if not os.path.isdir(participantdir): # to test on your own instance
- if os.path.isdir(participantdir):
- # if it does, it means that the webring is ALREADY added
- print_colors("[-] Webring Participant is already listed, skipping.")
- return False
- else:
- # if not, then proceed:
- # print_colors the URL to the csv files at http://URL.onion/participants/URL.onion/{verified.csv,unverified.csv,sensitive.csv,blacklist.csv,webring-participants.csv}
- basewurl='http://'+webring_participant_url+'/participants/'+webring_participant_url+'/'
- print_colors(basewurl)
- print_colors('[+] Checking if all of the required csv files exists for new webring participant ',webring_participant_url, ": ")
- w_vcsv=basewurl+'verified.csv'
- w_uvcsv=basewurl+'unverified.csv'
- #print_colors(CheckUrl(w_uvcsv))
- w_blcsv=basewurl+'blacklist.csv'
- #print_colors(CheckUrl(w_blcsv))
- w_scsv=basewurl+'sensitive.csv'
- #print_colors(CheckUrl(w_scsv))
- w_webcsv=basewurl+'webring-participants.csv'
- #print_colors(CheckUrl(w_webcsv))
- # verify that their verified.csv csv file exists at basewurl+'verified.csv'
- if CheckUrl(w_vcsv) is False or CheckUrl(w_uvcsv) is False or CheckUrl(w_blcsv) is False or CheckUrl(w_scsv) is False or CheckUrl(w_webcsv) is False:
- print_colors("[-] Webring Participant is invalid, exiting.")
- return False
- else:
- print_colors("[+] Webring Participant is valid, adding it.")
- name=''
- while(IsNameValid(name) is not True):
- name = input("What is the Webring instance name ? ")
- desc='DEFAULT'
- while(IsDescriptionValid(desc) is not True):
- desc=input("Description for the webring participant ? (Optional)")
- # if OK then add it to your own webring-participants.csv
- trusted=''
- status=''
- score=''
- newrow=[name,webring_participant_url,desc,trusted,status,score]
- webringcsvfile=instancepath+'/'+'webring-participants.csv'
- wdf = pd.read_csv(webringcsvfile)
- #print_colors("[+] NEWROW=",newrow)
- wdf.loc[-1] = newrow # adding a row
- wdf.index = wdf.index + 1 # shifting index
- wdf = wdf.sort_index() # sorting by index
- print_colors("[+] New row added! now writing the csv file:",webringcsvfile)
- wdf.to_csv(webringcsvfile, index=False)
- # create the directory in www/participants/PARTICIPANTURL/ if it's not there already
- if not os.path.exists(participantdir):
- os.makedirs(participantdir)
- # then download their csv files at http://URL.onion/participants/URL.onion/{verified.csv,unverified.csv,sensitive.csv,blacklist.csv,webring-participants.csv}
- # then save the csv file contents into a variable, then write it where it belongs:
- # for loop with each csv file you want:
- for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv']:
- # FOR EACH CSV FILE TO GET:
- # URL: basewurl / FILE.CSV
- # PATH: participantdir / FILE.CSV
- print_colors('[+] DOWNLOADING ',basewurl+i)
- # download the external csv file and save it into the "text" variable:
- #response = urllib.request.urlopen(basewurl+i)
- response = requests.get(basewurl+i, proxies=proxies)
- #data = response.read() # a `bytes` object
- #text = data.decode('utf-8')
- text = response.text
- # save the text variable into the destination file:
- print_colors('[+] SAVING IT INTO ',participantdir+'/'+i)
- csvfilepath=participantdir+'/'+i
- with open(csvfilepath, "w") as file:
- file.write(text)
- print_colors("[+] file written, let's read it")
- f = open(csvfilepath,"r")
- print_colors(f.read())
+ case 5:
+ print_colors("[+] Add a new webring participant (and download their files into their directory (without trusting them yet!))")
+ webring_participant_url = ''
+ while(IsOnionValid(webring_participant_url) is not True):
+ webring_participant_url = input("What is the onion domain of the new webring participant? (ex: lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion) ")
+ participantdir=rootpath+'www/participants/'+webring_participant_url
+ if os.path.isdir(participantdir):
+ print_colors("[-] Webring Participant is already listed, skipping.")
+ else:
+ basewurl='http://'+webring_participant_url+'/participants/'+webring_participant_url+'/'
+ print_colors(f"{basewurl}")
+ print_colors(f"[+] Checking if all of the required csv files exists for new webring participant {webring_participant_url} : ")
+ w_vcsv=basewurl+'verified.csv'
+ w_uvcsv=basewurl+'unverified.csv'
+ w_blcsv=basewurl+'blacklist.csv'
+ w_scsv=basewurl+'sensitive.csv'
+ w_webcsv=basewurl+'webring-participants.csv'
- # download the banner.png image:
+ # verify that their verified.csv csv file exists at basewurl+'verified.csv'
+ if CheckUrl(w_vcsv) is False or CheckUrl(w_uvcsv) is False or CheckUrl(w_blcsv) is False or CheckUrl(w_scsv) is False or CheckUrl(w_webcsv) is False:
+ print_colors("[-] Webring Participant is invalid, exiting.")
+ else:
+ print_colors("[+] Webring Participant is valid, adding it.")
+ name=''
+ while(IsNameValid(name) is not True):
+ name = input("What is the Webring instance name ? ")
+ desc='DEFAULT'
+ while(IsDescriptionValid(desc) is not True):
+ desc=input("Description for the webring participant ? (Optional)")
+ trusted=''
+ status=''
+ score=''
+ newrow=[name,webring_participant_url,desc,trusted,status,score]
+ webringcsvfile=instancepath+'/'+'webring-participants.csv'
+ wdf = pd.read_csv(webringcsvfile)
+ wdf.loc[-1] = newrow # adding a row
+ wdf.index = wdf.index + 1 # shifting index
+ wdf = wdf.sort_index() # sorting by index
+ print_colors(f"[+] New row added! now writing the csv file: {webringcsvfile}")
+ wdf.to_csv(webringcsvfile, index=False)
- bannerurl=basewurl+'banner.png'
- bannerpath=participantdir+'/banner.png'
- r = requests.get(bannerurl, stream=True, proxies=proxies)
- with open(bannerpath, 'wb') as f:
- r.raw.decode_content = True
- shutil.copyfileobj(r.raw, f)
+ if not os.path.exists(participantdir):
+ os.makedirs(participantdir)
+ for i in ['verified.csv','unverified.csv','blacklist.csv','sensitive.csv','webring-participants.csv']:
+ # FOR EACH CSV FILE TO GET:
+ # URL: basewurl / FILE.CSV
+ # PATH: participantdir / FILE.CSV
+ print_colors(f'[+] DOWNLOADING {basewurl}{i}')
+ response = requests.get(basewurl+i, proxies=proxies)
+ text = response.text
+ print_colors(f"[+] SAVING IT INTO participantdir/{i}")
+ csvfilepath=participantdir+'/'+i
+ with open(csvfilepath, "w") as file:
+ file.write(text)
+ print_colors("[+] file written, let's read it")
+ f = open(csvfilepath,"r")
+ print_colors(f.read())
- # SANITY CHECK ON THE BANNER PNG IMAGE:
- if IsBannerValid(bannerpath):
- print_colors('[+] Banner is valid')
- else:
- # if false, overwrite it with the template banner png file
- print_colors('[-] Banner is not valid, replacing it with the default banner')
- os.remove(bannerpath)
- # copy templates/banner.png to bannerpath
- bannertemplatepath=templatepath+'banner.png'
- shutil.copyfile(bannertemplatepath, bannerpath)
+ # download the banner.png image:
- ########### PERFORM SANITY CHECKS ON the webring participant's verified.csv and unverified.csv ##################
- for w in ['verified.csv','unverified.csv']:
- csvfilepath=participantdir+'/'+w
- csvdf = pd.read_csv(csvfilepath)
-
- #print_colors(bldf[['blacklisted-words']])
- bldf[['blacklisted-words']].iterrows()
- rows2delete= [] # it is an empty list at first
- for i,j in csvdf.iterrows():
- #print_colors("[+] Unverified.csv ROW=",i, uvdf.at[i, 'Instance'], uvdf.at[i, 'Category'], uvdf.at[i, 'Name'], uvdf.at[i, 'URL'], uvdf.at[i, 'Description'])
- #print_colors("[+] Unverified.csv ROW=",i, uvdf.iloc[[i]])
- #row=uvdf.iloc[[i]] #it displays the index
- row=csvdf.loc[i,:].values.tolist()
- #print_colors(i,row)
+ bannerurl=basewurl+'banner.png'
+ bannerpath=participantdir+'/banner.png'
+ r = requests.get(bannerurl, stream=True, proxies=proxies)
+ with open(bannerpath, 'wb') as f:
+ r.raw.decode_content = True
+ shutil.copyfileobj(r.raw, f)
+
+ # SANITY CHECK ON THE BANNER PNG IMAGE:
+ if IsBannerValid(bannerpath):
+ print_colors('[+] Banner is valid')
+ else:
+ # if false, overwrite it with the template banner png file
+ print_colors('[-] Banner is not valid, replacing it with the default banner')
+ os.remove(bannerpath)
+ # copy templates/banner.png to bannerpath
+ bannertemplatepath=templatepath+'banner.png'
+ shutil.copyfile(bannertemplatepath, bannerpath)
+
+ ########### PERFORM SANITY CHECKS ON the webring participant's verified.csv and unverified.csv ##################
+ for w in ['verified.csv','unverified.csv']:
+ csvfilepath=participantdir+'/'+w
+ csvdf = pd.read_csv(csvfilepath)
+
+ #print_colors(bldf[['blacklisted-words']])
+ bldf[['blacklisted-words']].iterrows()
+ rows2delete= [] # it is an empty list at first
+ for i,j in csvdf.iterrows():
+ #row=uvdf.iloc[[i]] #it displays the index
+ row=csvdf.loc[i,:].values.tolist()
+
+ ################################ SANITY CHECKS ####################################
+ ### SANITY CHECK 0: make sure that ✔️ and x are replaced with YES/NO, as it changed since v1.0.1 ###
+ if csvdf.at[i, 'Status'] == "✔️" or csvdf.at[i, 'Status'] == "YES" :
+ csvdf.at[i, 'Status'] = "YES"
+ csvdf.to_csv(csvfilepath, index=False)
+ else:
+ csvdf.at[i, 'Status'] = "NO"
+ csvdf.to_csv(csvfilepath, index=False)
+
+ if csvdf.at[i, 'Sensitive'] == "✔️" or csvdf.at[i, 'Sensitive'] == "YES" :
+ csvdf.at[i, 'Sensitive'] = "YES"
+ csvdf.to_csv(csvfilepath, index=False)
+ else:
+ csvdf.at[i, 'Sensitive'] = "NO"
+ csvdf.to_csv(csvfilepath, index=False)
- ### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion###
- #print_colors("[+] ROW=",i,"ROW CONTENTS=", IsUrlValid(csvdf.at[i, 'Instance']), IsCategoryValid(csvdf.at[i, 'Category']), IsNameValid(csvdf.at[i, 'Name']), IsUrlValid(csvdf.at[i, 'URL']), IsStatusValid(csvdf.at[i, 'Sensitive']), IsDescriptionValid(csvdf.at[i, 'Description']), IsStatusValid(csvdf.at[i, 'Status']), IsScoreValid(csvdf.at[i, 'Score']))
- if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False:
- #mark the row for deletion as it has invalid inputs
- if i not in rows2delete:
- print_colors("Marking row", i,"for deletion, as it has invalid inputs")
- rows2delete.append(i) #mark the row for deletion if not already done
-
- ### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ###
- for k,l in bldf.iterrows():
- #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words'])
- blword=bldf.at[k, 'blacklisted-words']
- if any(blword in str(x) for x in row) == True:
- #print_colors("found blacklisted word! marking row for deletion")
- if i not in rows2delete:
- print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word")
- rows2delete.append(i) #mark the row for deletion if not already done
+ ### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion###
+ if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False:
+ #mark the row for deletion as it has invalid inputs
+ if i not in rows2delete:
+ print_colors(f"Marking row {i} for deletion, as it has invalid inputs")
+ rows2delete.append(i) #mark the row for deletion if not already done
- ### SANITY CHECK 3: Mark all the rows that are supposed to be sensitive ###
- for k,l in sedf.iterrows():
- #print_colors("[+] Sensitive word=",k, sedf.at[k, 'sensitive-words'])
- seword=sedf.at[k, 'sensitive-words']
- if any(seword in str(x) for x in row) == True:
- if csvdf.at[i, 'Sensitive'] != '✔️':
- print_colors("Marking row", i,"as sensitive, as it matches with a sensitive word")
- csvdf.at[i, 'Sensitive']='✔️'
+ ### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ###
+ for k,l in bldf.iterrows():
+ blword=bldf.at[k, 'blacklisted-words']
+ if any(blword in str(x) for x in row) == True:
+ if i not in rows2delete:
+ print_colors(f"Marking row {i} for deletion, as it matches with a blacklisted word")
+ rows2delete.append(i) #mark the row for deletion if not already done
- print_colors(f'[-] Rows to delete: {rows2delete}')
-
- for i in rows2delete:
- row=csvdf.loc[i,:].values.tolist()
- print_colors('[+] REMOVING ROW :',i,row)
- csvdf.drop(i, inplace= True)
- csvdf.to_csv(csvfilepath, index=False)
+ ### SANITY CHECK 3: Mark all the rows that are supposed to be sensitive ###
+ for k,l in sedf.iterrows():
+ seword=sedf.at[k, 'sensitive-words']
+ if any(seword in str(x) for x in row) == True:
+ if csvdf.at[i, 'Sensitive'] != 'NO':
+ print_colors(f"Marking row {i} as sensitive, as it matches with a sensitive word")
+ csvdf.at[i, 'Sensitive']='YES'
+
+ print_colors(f"[-] Rows to delete: {rows2delete}")
+
+ for i in rows2delete:
+ row=csvdf.loc[i,:].values.tolist()
+ print_colors(f"[+] REMOVING ROW: {i}{row}")
+ csvdf.drop(i, inplace= True)
+ csvdf.to_csv(csvfilepath, index=False)
##############################################
@@ -643,123 +598,121 @@ Maintenance:
-
- case "6":
- print_colors("[+] Trust/UnTrust/Blacklist a webring participant (Potentially dangerous)")
- webringcsvfile=instancepath+'/'+'webring-participants.csv'
- wdf = pd.read_csv(webringcsvfile)
- # list each webring participant in your webring-participants.csv file
- print_colors(wdf[["URL","Trusted"]])
- # ask the user to pick an index
- index=""
- while (index not in wdf.index):
- # prompt the user to ask for with row they want to move to verified.csv
- index = int(input("What is the index of the webring participant that you want to edit ? (ex: 3) "))
- # once a valid index is picked, ask if the user wants to 1) trust the webring participant, or 2) untrust them, or 3) black list them
- choice=""
- while (choice not in ["1","2","3"]):
- choice = input("Do you want to 1) Trust, 2) UnTrust, or 3) Blacklist the webring participant ?")
- if choice == "1":
- # trust the webring participant
- # ask the user if they want to proceed, as this is potentially risky if the webring participant tries to list malicious links in the future
- choice2=input("You're about to trust another peer, this means that you're going to automatically trust all of the links they have in their verified.csv file! If this is a malicious peer, you're about to potentially going to automatically trust malicious links, it is potentially risky! Do you want to continue ? (y/n)")
- if choice2 == "y":
- # if user wants to proceed, mark the "trusted" column as V
- print_colors("[+] Trusting webring participant", wdf.at[index,"URL"])
- wdf.at[index,"Trusted"]='✔️'
- wdf.to_csv(webringcsvfile, index=False)
-
- else:
- print_colors("[-] not trusting webring participant, skipping.")
- if choice == "2":
- print_colors("[+] UnTrusting webring participant", wdf.at[index,"URL"])
- wdf.at[index,"Trusted"]=''
- wdf.to_csv(webringcsvfile, index=False)
- # untrust the webring participant
- # if 2: mark the "trusted" column as empty
- if choice == "3":
- print_colors("[+] Blacklisting webring participant", wdf.at[index,"URL"])
- # blacklist the webring participant
- # add it's URL to your own blacklist.csv
- instance2blacklist=wdf.at[index,"URL"]
- newrow=[instance2blacklist]
- print_colors("[+] NEWROW=",newrow)
- # (rest is automatic: status, score, instance is = '' because it is your own instance)
- # check if the entry doesn't already exist in verified.csv and in unverified.csv
- # if it doesnt exist, add it into unverified.csv
- bldf.loc[-1] = newrow # adding a row
- bldf.index = bldf.index + 1 # shifting index
- bldf = bldf.sort_index() # sorting by index
- print_colors("[+] New row added! now writing the csv file:")
- bldf.to_csv(blcsvfile, index=False)
- # remove all of the entries that came from that participant (drop the lines in your own verified+unverified.csv that have that instance in the instance column)
+ case 6:
+ while True:
+ print_colors("[+] Trust/UnTrust/Blacklist a webring participant (Potentially dangerous)")
+ webringcsvfile=instancepath+'/'+'webring-participants.csv'
+ wdf = pd.read_csv(webringcsvfile)
+ print_colors(f'{wdf[["URL","Trusted"]]}')
+ try:
+ index = int(input("What is the index of the webring participant that you want to edit? -1 to exit ").strip())
+ if index == -1:
+ break
+ elif index in wdf.index:
+ choice = int(input("Do you want to 1) Trust, 2) UnTrust, or 3) Blacklist the webring participant?").strip())
+ while True:
+ match choice:
+ case 1:
+ # trust the webring participant
+ choice2=input("You're about to trust another peer, this means that you're going to automatically trust all of the links they have in their verified.csv file! If this is a malicious peer, this action might be potentially risky! Do you want to continue ? (y/n)")
+ if choice2 == "y":
+ print_colors(f'[+] Trusting webring participant {wdf.at[index,"URL"]}')
+ ## Warning: In future versions of panda '✔️' will not work. It will show an error.
+ wdf.at[index,"Trusted"]= 'YES'
+ wdf.to_csv(webringcsvfile, index=False)
+ break
+ else:
+ print_colors("[-] not trusting webring participant", is_error=True)
+ break
- #vdf
- rows2delete= [] # it is an empty list at first
- for i,j in vdf.iterrows():
- row=vdf.loc[i,:].values.tolist()
- for k,l in bldf.iterrows():
- #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words'])
- blword=bldf.at[k, 'blacklisted-words']
- if any(blword in str(x) for x in row) == True:
- #print_colors("found blacklisted word! marking row for deletion")
- if i not in rows2delete:
- print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word")
- rows2delete.append(i) #mark the row for deletion if not already done
- for i in rows2delete:
- row=vdf.loc[i,:].values.tolist()
- print_colors('[+] REMOVING ROW :',i,row)
- vdf.drop(i, inplace= True)
- vdf.to_csv(verifiedcsvfile, index=False)
- print_colors(vdf)
- rows2delete= [] # it is an empty list at first
- #uvdf
- rows2delete= [] # it is an empty list at first
- for i,j in uvdf.iterrows():
- row=uvdf.loc[i,:].values.tolist()
- for k,l in bldf.iterrows():
- #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words'])
- blword=bldf.at[k, 'blacklisted-words']
- if any(blword in str(x) for x in row) == True:
- #print_colors("found blacklisted word! marking row for deletion")
- if i not in rows2delete:
- print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word")
- rows2delete.append(i) #mark the row for deletion if not already done
- for i in rows2delete:
- row=uvdf.loc[i,:].values.tolist()
- print_colors('[+] REMOVING ROW :',i,row)
- uvdf.drop(i, inplace= True)
- uvdf.to_csv(unverifiedcsvfile, index=False)
- print_colors(uvdf)
- rows2delete= [] # it is an empty list at first
+ case 2:
+ print_colors(f'[+] UnTrusting webring participant {wdf.at[index,"URL"]}')
+ ## Warning: In future versions of panda '' will not work. It will show an error. Maybe change to a 0,1
+ wdf.at[index,"Trusted"]='NO'
+ wdf.to_csv(webringcsvfile, index=False)
+ break
- # find all rows that match with the instance name in wdf aswell to remove them
- for i,j in wdf.iterrows():
- row=wdf.loc[i,:].values.tolist()
- for k,l in bldf.iterrows():
- #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words'])
- blword=bldf.at[k, 'blacklisted-words']
- if any(blword in str(x) for x in row) == True:
- #print_colors("found blacklisted word! marking row for deletion")
- if i not in rows2delete:
- print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word")
- rows2delete.append(i) #mark the row for deletion if not already done
- for i in rows2delete:
- row=wdf.loc[i,:].values.tolist()
- print_colors('[+] REMOVING ROW :',i,row)
- wdf.drop(i, inplace= True)
- wdf.to_csv(webringcsvfile, index=False)
- print_colors(wdf)
- rows2delete= [] # it is an empty list at first
+ case 3:
+ print_colors(f'[+] Blacklisting webring participant {wdf.at[index,"URL"]}')
+ instance2blacklist=wdf.at[index,"URL"]
+ newrow=[instance2blacklist]
+ print_colors(f"[+] NEWROW= {newrow}")
+ # (rest is automatic: status, score, instance is = '' because it is your own instance)
+ # check if the entry doesn't already exist in verified.csv and in unverified.csv
+ # if it doesnt exist, add it into unverified.csv
+ bldf.loc[-1] = newrow # adding a row
+ bldf.index = bldf.index + 1 # shifting index
+ bldf = bldf.sort_index() # sorting by index
+ print_colors("[+] New row added! now writing the csv file:")
+ bldf.to_csv(blcsvfile, index=False)
+
+
+ # remove all of the entries that came from that participant (drop the lines in your own verified+unverified.csv that have that instance in the instance column)
+
+ rows2delete= [] # it is an empty list at first
+ for i,j in vdf.iterrows():
+ row=vdf.loc[i,:].values.tolist()
+ for k,l in bldf.iterrows():
+ blword=bldf.at[k, 'blacklisted-words']
+ if any(blword in str(x) for x in row) == True:
+ if i not in rows2delete:
+ print_colors(f"Marking row {i} for deletion, as it matches with a blacklisted word")
+ rows2delete.append(i) #mark the row for deletion if not already done
+ for i in rows2delete:
+ row=vdf.loc[i,:].values.tolist()
+ print_colors(f'[+] REMOVING ROW: {i} {row}')
+ vdf.drop(i, inplace= True)
+ vdf.to_csv(verifiedcsvfile, index=False)
+ print_colors(f"{vdf}")
+ rows2delete= [] # it is an empty list at first
+ rows2delete= [] # it is an empty list at first
+ for i,j in uvdf.iterrows():
+ row=uvdf.loc[i,:].values.tolist()
+ for k,l in bldf.iterrows():
+ blword=bldf.at[k, 'blacklisted-words']
+ if any(blword in str(x) for x in row) == True:
+ if i not in rows2delete:
+ print_colors(f"Marking row {i} for deletion, as it matches with a blacklisted word")
+ rows2delete.append(i) #mark the row for deletion if not already done
+ for i in rows2delete:
+ row=uvdf.loc[i,:].values.tolist()
+ print_colors(f'[+] REMOVING ROW: {i} {row}')
+ uvdf.drop(i, inplace= True)
+ uvdf.to_csv(unverifiedcsvfile, index=False)
+ print_colors(f"{uvdf}")
+ rows2delete= [] # it is an empty list at first
+
+ # find all rows that match with the instance name in wdf aswell to remove them
+ for i,j in wdf.iterrows():
+ row=wdf.loc[i,:].values.tolist()
+ for k,l in bldf.iterrows():
+ blword=bldf.at[k, 'blacklisted-words']
+ if any(blword in str(x) for x in row) == True:
+ if i not in rows2delete:
+ print_colors(f"Marking row {i} for deletion, as it matches with a blacklisted word")
+ rows2delete.append(i) #mark the row for deletion if not already done
+ for i in rows2delete:
+ row=wdf.loc[i,:].values.tolist()
+ print_colors(f'[+] REMOVING ROW: {i} {row}')
+ wdf.drop(i, inplace= True)
+ wdf.to_csv(webringcsvfile, index=False)
+ print_colors(f"{wdf}")
+ rows2delete= [] # it is an empty list at first
- # remove the entire directory in www/participants/INSTANCENAME aswell to get rid of it
- instance2blacklistpath=rootpath+'www/participants/'+instance2blacklist
- print_colors("[+] removing the participant's directory at ",instance2blacklistpath)
- shutil.rmtree(instance2blacklistpath)
+ # remove the entire directory in www/participants/INSTANCENAME aswell to get rid of it
+ instance2blacklistpath=rootpath+'www/participants/'+instance2blacklist
+ print_colors(f"[+] removing the participant's directory at {instance2blacklistpath}")
+ shutil.rmtree(instance2blacklistpath)
+
+ case _:
+ break
+ except Exception:
+ break
+
@@ -772,210 +725,237 @@ Maintenance:
#Maintenance:
# 9) remove the duplicate URLs for your own instance
-# 10) perform sanity checks on all csv files (all instances) (to mark them as sensitive / or remove the ones that are blacklisted)
+# 10) perform sanity checks on all csv files (all instances) (to mark them as sensitive / or remove the ones that are blacklisted)
#########################################################
- case "7":
- print_colors("[+] Add/Remove Words/URLs in the sensitive list (ex: drug)")
- #secsvfile=instancepath+'/sensitive.csv' #fyi
- #sedf = pd.read_csv(secsvfile) #fyi
- option="0"
-
- done = False
- while(done == False):
- while option != "1" and option != "2" and option != "-1":
- option=input("[+] do you want to 1) add or 2) remove Words/URLs? (type exit to exit) ")
- if option == "1":
- word=input("[+] which Sensitive word do you want to add? (write -1 to exit) ")
- if word == "-1":
- done = True
- #True to get out of the while loop
- else:
- print_colors("[+] checking if the Word/URL is valid: ")
- if IsUrlValid(word) or IsOnionValid(word) or IsDescriptionValid(word):
- print_colors(IsUrlValid(word), IsOnionValid(word), IsDescriptionValid(word))
- print_colors('[+] Word/URL is valid, adding the word into the sensitive wordlist ')
- # add it to the sensitive wordlist
- newrow=[word]
- print_colors("[+] NEWROW=",newrow)
- # (rest is automatic: status, score, instance is = '' because it is your own instance)
- # check if the entry doesn't already exist in verified.csv and in unverified.csv
- # if it doesnt exist, add it into unverified.csv
- sedf.loc[-1] = newrow # adding a row
- sedf.index = sedf.index + 1 # shifting index
- sedf = sedf.sort_index() # sorting by index
- print_colors("[+] New row added! now writing the csv file: ")
- sedf.to_csv(secsvfile, index=False)
-
-
- if option == "2":
- print_colors(sedf)
- index=""
- while (index not in sedf.index) and index != -1:
- index=int(input("which word do you want to remove? (index 0 to (max index) (write -1 to exit) "))
- if index == -1:
- done = True
- #True to get out of the while loop
- else:
- if (index in sedf.index):
- #if index exists, remove it
- print_colors("[+] removing selected index: ")
- sedf.drop(index, inplace= True)
- sedf.to_csv(secsvfile, index=False)
-
- else:
- print_colors('[-] Error, invalid index')
-
-
- else:
- pass
-
-
+ case 7:
+ print_colors("[+] Add/Remove Words/URLs in the sensitive list (ex: drug)")
+ try:
+ option = int(input("[+] do you want to 1) add or 2) remove Words/URLs? (type -1 to exit) "))
+ match option:
+ case 1:
+ while True:
+ word=input("[+] which Sensitive word/url do you want to add? (write -1 to exit) ")
+ if word == "-1":
+ break
+ else:
+ print_colors("[+] checking if the Word/URL is valid: ")
+ if IsUrlValid(word) or IsOnionValid(word) or IsDescriptionValid(word):
+ print_colors('[+] Word/URL is valid, adding the word into the sensitive wordlist')
+ newrow=[word]
+ print_colors(f"[+] NEWROW= {newrow}")
+ sedf.loc[-1] = newrow
+ sedf.index = sedf.index + 1
+ sedf = sedf.sort_index()
+ print_colors("[+] New row added! now writing the csv file.")
+ sedf.to_csv(secsvfile, index=False)
-
- case "8":
- print_colors("[+] Add/Remove words in the blacklist list (ex: porn)")
- #blcsvfile=instancepath+'/sensitive.csv' #fyi
- #bldf = pd.read_csv(secsvfile) #fyi
- option="0"
-
- done = False
- while(done == False):
- while option != "1" and option != "2" and option != "-1":
- option=input("[+] Do you want to 1) add or 2) remove Words/URLs? (type exit to exit) ")
- if option == "1":
- word=input("[+] Which Sensitive word do you want to add? (write -1 to exit) ")
- if word == "-1":
- done = True
- #True to get out of the while loop
- else:
- print_colors("[+] Checking if the Word/URL is valid: ")
- if IsUrlValid(word) or IsOnionValid(word) or IsDescriptionValid(word):
- print_colors(IsUrlValid(word), IsOnionValid(word), IsDescriptionValid(word))
- print_colors('[+] Word/URL is valid, adding the word into the blacklist ')
- # add it to the sensitive wordlist
- newrow=[word]
- print_colors("[+] NEWROW=",newrow)
- # (rest is automatic: status, score, instance is = '' because it is your own instance)
- # check if the entry doesn't already exist in verified.csv and in unverified.csv
- # if it doesnt exist, add it into unverified.csv
- bldf.loc[-1] = newrow # adding a row
- bldf.index = bldf.index + 1 # shifting index
- bldf = bldf.sort_index() # sorting by index
- print_colors("[+] New row added! now writing the csv file: ")
- bldf.to_csv(blcsvfile, index=False)
-
-
- if option == "2":
- print_colors(bldf)
- index=""
- while (index not in bldf.index) and index != -1:
- index=int(input("which word do you want to remove? (index 0 to (max index) (write -1 to exit) "))
- if index == -1:
- done = True
- #True to get out of the while loop
- else:
- if (index in bldf.index):
- #if index exists, remove it
- print_colors("[+] removing selected index: ")
- bldf.drop(index, inplace= True)
- bldf.to_csv(blcsvfile, index=False)
-
- else:
- print_colors('[-] Error, invalid index')
-
-
- else:
- pass
-
-
+ case 2:
+ while True:
+ print_colors(f"{sedf}")
+ index=input("which word do you want to remove? (index 0 to (max index) (write -1 to exit) ")
+ try:
+ indices = index.split(' ')
+ if len(indices) == 2:
+ for i in range(int(indices[0]),int(indices[1])):
+ try:
+ idx = int(i)
+ if idx in sedf.index:
+ print_colors("[+] removing selected index.")
+ sedf.drop(index=idx, inplace=True)
+ sedf.to_csv(secsvfile, index=True)
+ else:
+ print_colors(f"[-] Index {idx} does not exist.", is_error=True)
+ except ValueError:
+ print_colors(f"[-] Error: '{i}' is not a valid integer.", is_error=True)
+ elif len(indices) == 1:
+ try:
+ idx = int(indices[0])
+ if idx != -1:
+ if idx in sedf.index:
+ print_colors("[+] removing selected index.")
+ sedf.drop(idx, inplace=True)
+ sedf.to_csv(secsvfile, index=True)
+ else:
+ print_colors(f"[-] Index {idx} does not exist.", is_error=True)
+ elif idx == -1:
+ break
+ except ValueError:
+ print_colors(f"[-] Error: '{indices[0]}' is not a valid integer.", is_error=True)
+ else:
+ print_colors('[-] Error, invalid index', is_error=True)
+ except Exception as e:
+ print_colors(f"[-] An unexpected error occurred: {str(e)}", is_error=True)
+ except Exception:
+ break
- # CASE 9 : cleanup all duplicates in unverified + verified.csv, based on the url (check if each url appears more than once, and if they do, remove them + write to csv file)
- case "9":
- print_colors("[+] 9) Cleaning up all duplicates in your own unverified + verified.csv (based on the url)")
- # ignore it if the index is "indextocheck" and if the index is already listed in rows2delete
- # else: add the index to "rows2delete"
- # go drop the rows by their index listed in "rows2delete"
-################################ CHECKING FOR DUPLICATES! #########################
- # for unverified.csv, and verified.csv
- for w in ['verified.csv','unverified.csv']:
- #instancepath=rootpath+'www/participants/'+instance # fyi
- csvfilepath=instancepath+'/'+w
-
- print_colors(csvfilepath)
- csvdf = pd.read_csv(csvfilepath)
- print_colors("REMOVING DUPLICATES IN", csvfilepath)
- csvdf = csvdf.drop_duplicates(subset=['URL'])
- csvdf.to_csv(csvfilepath, index=False)
- print_colors(csvdf[['URL']])
- case "10":
- print_colors("[+] 10) perform sanity checks on all csv files (to mark them as sensitive / or remove the ones that are blacklisted)")
- participantspath = rootpath+'www/participants/'
- for participant in os.listdir(participantspath):
- print_colors("Participant:",participant)
- participantdir= participantspath+participant
- a=0
- if a == 0:
- if a== 0:
- ################ BEGIN SANITY CHECKS FOR EACH PARTICIPANTS ##############
- # iterate through the participant's verified.csv and unverified.csv files
- for w in ['verified.csv','unverified.csv']:
- csvfilepath=participantdir+'/'+w
- print_colors(csvfilepath)
- csvdf = pd.read_csv(csvfilepath)
- #print_colors(bldf[['blacklisted-words']])
- bldf[['blacklisted-words']].iterrows()
- rows2delete= [] # it is an empty list at first
- for i,j in csvdf.iterrows():
- #print_colors("[+] Unverified.csv ROW=",i, uvdf.at[i, 'Instance'], uvdf.at[i, 'Category'], uvdf.at[i, 'Name'], uvdf.at[i, 'URL'], uvdf.at[i, 'Description'])
- #print_colors("[+] Unverified.csv ROW=",i, uvdf.iloc[[i]])
- #row=uvdf.iloc[[i]] #it displays the index
- row=csvdf.loc[i,:].values.tolist()
- print_colors(row)
- #print_colors(i,row)
+
+ case 8:
+ print_colors("[+] Add/Remove words in the blacklist list (ex: porn)")
+ try:
+ option= int(input("[+] Do you want to 1) add or 2) remove Words/URLs? (type -1 to exit) "))
+
+
+ match option:
+ case 1:
+ while True:
+ word=input("[+] Which Sensitive word do you want to add? (write -1 to exit) ")
+ if word == "-1":
+ break
+ else:
+ print_colors("[+] Checking if the Word/URL is valid: ")
+ if IsUrlValid(word) or IsOnionValid(word) or IsDescriptionValid(word):
+ print_colors('[+] Word/URL is valid, adding the word into the blacklist')
+ newrow=[word]
+ print_colors(f"[+] NEWROW= {newrow}")
+ # (rest is automatic: status, score, instance is = '' because it is your own instance)
+ # check if the entry doesn't already exist in verified.csv and in unverified.csv
+ # if it doesnt exist, add it into unverified.csv
+ bldf.loc[-1] = newrow # adding a row
+ bldf.index = bldf.index + 1 # shifting index
+ bldf = bldf.sort_index() # sorting by index
+ print_colors("[+] New row added! Now writing the csv file")
+ bldf.to_csv(blcsvfile, index=False)
+
+
+ case 2:
+ while True:
+ print_colors(f"{bldf}")
+ index=input("which word do you want to remove? (index 0 to (max index) (write -1 to exit) ").strip()
+ try:
+ indices = index.split(' ')
+ if len(indices) == 2:
+ for i in range(int(indices[0]),int(indices[1])):
+ try:
+ idx = int(i)
+ if idx in bldf.index:
+ print_colors("[+] removing selected index.")
+ bldf.drop(index=idx, inplace=True)
+ bldf.to_csv(blcsvfile, index=True)
+ else:
+ print_colors(f"[-] Index {idx} does not exist.", is_error=True)
+ except ValueError:
+ print_colors(f"[-] Error: '{i}' is not a valid integer.", is_error=True)
+ elif len(indices) == 1:
+ try:
+ idx = int(indices[0])
+ if idx != -1:
+ if idx in bldf.index:
+ print_colors("[+] removing selected index.")
+ bldf.drop(idx, inplace=True)
+ bldf.to_csv(blcsvfile, index=False)
+ else:
+ print_colors(f"[-] Index {idx} does not exist.", is_error=True)
+ elif idx == -1:
+ break
+ except ValueError:
+ print_colors(f"[-] Error: '{indices[0]}' is not a valid integer.", is_error=True)
+ else:
+ print_colors('[-] Error, invalid index', is_error=True)
+ except Exception as e:
+ print_colors(f"[-] An unexpected error occurred: {str(e)}", is_error=True)
+ except Exception:
+ break
- ################################ SANITY CHECKS ####################################
- ### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion###
- #print_colors("[+] ROW=",i,"ROW CONTENTS=", IsUrlValid(uvdf.at[i, 'Instance']), IsCategoryValid(uvdf.at[i, 'Category']), IsNameValid(uvdf.at[i, 'Name']), IsUrlValid(uvdf.at[i, 'URL']), IsStatusValid(uvdf.at[i, 'Sensitive']), IsDescriptionValid(uvdf.at[i, 'Description']), IsStatusValid(uvdf.at[i, 'Status']), IsScoreValid(uvdf.at[i, 'Score']))
- if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False:
- #mark the row for deletion as it has invalid inputs
- if i not in rows2delete:
- print_colors("Marking row", i,"for deletion, as it has invalid inputs")
- rows2delete.append(i) #mark the row for deletion if not already done
-
- ### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ###
- for k,l in bldf.iterrows():
- #print_colors("[+] Blacklisted word=",k, bldf.at[k, 'blacklisted-words'])
- blword=bldf.at[k, 'blacklisted-words']
- if any(blword in str(x) for x in row) == True:
- #print_colors("found blacklisted word! marking row for deletion")
- if i not in rows2delete:
- print_colors("Marking row", i,"for deletion, as it matches with a blacklisted word")
- rows2delete.append(i) #mark the row for deletion if not already done
-
- for i in rows2delete:
- row=csvdf.loc[i,:].values.tolist()
- print_colors('[+] REMOVING ROW :',i,row)
- csvdf.drop(i, inplace= True)
- csvdf.to_csv(csvfilepath, index=False)
+ case 9:
+ print_colors("[+] 9) Cleaning up all duplicates in your own unverified + verified.csv (based on the url)")
+ for w in ['verified.csv', 'unverified.csv']:
+ csvfilepath = os.path.join(instancepath, w)
+ print_colors(f"Processing file: {csvfilepath}")
+ try:
+ csvdf = pd.read_csv(csvfilepath)
+ print_colors(f"Removing duplicates in {csvfilepath}")
+ csvdf = csvdf.drop_duplicates(subset=['URL'])
+ csvdf.to_csv(csvfilepath, index=False)
+ print_colors(f"Cleaned data:\n{csvdf[['URL']]}")
+ except FileNotFoundError:
+ print_colors(f"File not found: {csvfilepath}")
+ except Exception as e:
+ print_colors(f"An error occurred while processing {csvfilepath}: {e}")
+ break
+
+ case 10:
+ print_colors("[+] 10) perform sanity checks on all csv files (to mark them as sensitive / or remove the ones that are blacklisted)")
+ participantspath = rootpath+'www/participants/'
+ for participant in os.listdir(participantspath):
+ print_colors(f"Participant: {participant}")
+ #read=input("Continue?")
+ participantdir= participantspath+participant
+ ################ BEGIN SANITY CHECKS FOR EACH PARTICIPANTS ##############
+ # iterate through the participant's verified.csv and unverified.csv files
+ for w in ['verified.csv','unverified.csv']:
+ csvfilepath=participantdir+'/'+w
+ print_colors(f"{csvfilepath}")
+ csvdf = pd.read_csv(csvfilepath)
+ rows2delete= [] # it is an empty list at first
+ for i,j in csvdf.iterrows():
+ row=csvdf.loc[i,:].values.tolist()
+ print_colors(f"{row}")
- # TODO find the list of all csv files (in www/participants/*/*.csv) (templates should remain empty by default)
- # copy what was done in option 4, to :
- # delete the ones that have invalid entries
- # mark the sensitive rows as sensitive
- # delete the rows that match with blacklisted words
- case _:
- print_colors("[-] Exiting")
- return True
+ ################################ SANITY CHECKS ####################################
+ ### SANITY CHECK 0: make sure that ✔️ and x are replaced with YES/NO, as it changed since v1.0.1 ###
+ if csvdf.at[i, 'Status'] == "✔️" or csvdf.at[i, 'Status'] == "YES" :
+ csvdf.at[i, 'Status'] = "YES"
+ csvdf.to_csv(csvfilepath, index=False)
+ else:
+ csvdf.at[i, 'Status'] = "NO"
+ csvdf.to_csv(csvfilepath, index=False)
+
+ if csvdf.at[i, 'Sensitive'] == "✔️" or csvdf.at[i, 'Sensitive'] == "YES" :
+ csvdf.at[i, 'Sensitive'] = "YES"
+ csvdf.to_csv(csvfilepath, index=False)
+ else:
+ csvdf.at[i, 'Sensitive'] = "NO"
+ csvdf.to_csv(csvfilepath, index=False)
+
+ ### SANITY CHECK 1: Mark all the rows that have incorrect formatting for deletion###
+ if IsUrlValid(csvdf.at[i, 'Instance']) is False or IsCategoryValid(csvdf.at[i, 'Category']) is False or IsNameValid(csvdf.at[i, 'Name']) is False or IsUrlValid(csvdf.at[i, 'URL']) is False or IsStatusValid(csvdf.at[i, 'Sensitive']) is False or IsDescriptionValid(csvdf.at[i, 'Description']) is False or IsStatusValid(csvdf.at[i, 'Status']) is False or IsScoreValid(csvdf.at[i, 'Score']) is False:
+ if i not in rows2delete:
+ print_colors(f"Marking row {i} for deletion, as it has invalid inputs")
+ print_colors(f"{row}")
+ print(IsUrlValid(csvdf.at[i, 'Instance']), IsCategoryValid(csvdf.at[i, 'Category']), IsNameValid(csvdf.at[i, 'Name']), IsUrlValid(csvdf.at[i, 'URL']), IsStatusValid(csvdf.at[i, 'Sensitive']), IsDescriptionValid(csvdf.at[i, 'Description']), IsStatusValid(csvdf.at[i, 'Status']), IsScoreValid(csvdf.at[i, 'Score']))
+ rows2delete.append(i)
+ read=input("Continue?")
+
+ ### SANITY CHECK 2: Mark all rows that are not allowed (blacklist) for deletion ###
+ for k,l in bldf.iterrows():
+ blword=bldf.at[k, 'blacklisted-words']
+ if any(blword in str(x) for x in row) == True:
+ if i not in rows2delete:
+ print_colors(f"Marking row {i} for deletion, as it matches with the blacklisted word {blword}")
+ rows2delete.append(i)
+ #read=input("Continue?")
+ ### SANITY CHECK 3: Mark all rows that match sensitive words to be sensitive = YES
+ for k,l in sedf.iterrows():
+ seword=sedf.at[k, 'sensitive-words']
+ if any(seword in str(x) for x in row) == True:
+ print_colors(f"Marking row {i} as sensitive, as it matches with the sensitive word {seword}")
+ csvdf.at[i, 'Sensitive']="YES"
+ csvdf.to_csv(csvfilepath, index=False)
+ #read=input("Continue?")
+
+
+ for i in rows2delete:
+ row=csvdf.loc[i,:].values.tolist()
+ print_colors(f'[+] REMOVING ROW : {i} {row}')
+ csvdf.drop(i, inplace= True)
+ csvdf.to_csv(csvfilepath, index=False)
+ #read=input("Continue?")
+
+ case 0:
+ print_colors(f"[-] Exiting", bold=True)
+ break
+
+
if __name__ == '__main__':
- main()
+ main()
diff --git a/scripts/uptimechecker.py b/scripts/uptimechecker.py
index 4d87054..d1b929e 100644
--- a/scripts/uptimechecker.py
+++ b/scripts/uptimechecker.py
@@ -1,4 +1,4 @@
-import os,pwd,re
+import os,re,pwd
import csv
import requests
import json
@@ -13,6 +13,7 @@ def main():
# TODO get the instance name and exit if its not there
rootpath='/srv/darknet-lantern/'
urlpath=pwd.getpwuid(os.getuid()).pw_dir+"/.darknet_participant_url"
+
#print(urlpath)
@@ -24,7 +25,6 @@ def main():
with open(urlpath) as f:
instance = f.read().rstrip()
# check if the instance URL domain is valid
- #print(urlpath,instance)
if IsOnionValid(instance):
print("[+] Instance Name:",instance,IsOnionValid(instance))
isitvalid="y"
@@ -35,7 +35,6 @@ def main():
print("[-] Instance path doesn't exist yet, run darknet_exploration.py to set it up" )
return False
- #i=input("continue?")
proxies = {
'http': 'socks5h://127.0.0.1:9050',
'https': 'socks5h://127.0.0.1:9050'
@@ -72,38 +71,37 @@ def main():
print('[+]',url,status)
if status != 502:
print(url,"✔️")
- df.at[i,"Status"]="✔️"
+ df.at[i,"Status"]="YES"
#if uptime <100 do +1 to the value
if df.at[i,"Score"] < 100:
df.at[i,"Score"] = df.at[i,"Score"] + 1
else:
print(url,"❌")
- df.at[i,"Status"]="❌"
+ df.at[i,"Status"]="NO"
#if uptime >0 do -1 to the value
if df.at[i,"Score"] > 0:
df.at[i,"Score"] = df.at[i,"Score"] - 1
except requests.ConnectionError as e:
#print(e)
print(url,"❌")
- df.at[i,"Status"]="❌"
+ df.at[i,"Status"]="NO"
#if uptime >0 do -1 to the value
if df.at[i,"Score"] > 0:
df.at[i,"Score"] = df.at[i,"Score"] - 1
except requests.exceptions.ReadTimeout as e:
#print(e)
print(url,"❌")
- df.at[i,"Status"]="❌"
+ df.at[i,"Status"]="NO"
#if uptime >0 do -1 to the value
if df.at[i,"Score"] > 0:
df.at[i,"Score"] = df.at[i,"Score"] - 1
+ df2 = df.sort_values(by=["Score"], ascending=False)
#sort by category if you are verified/unverified.csv
if csvfilename in csvfiles2sortcat:
- df2 = df.sort_values(by=["Category","Name"], ascending=[True,True])
- else:
- df2 = df.sort_values(by="Score", ascending=False)
+ df2 = df.sort_values(by=["Category"], ascending=True)
+ #print(df2)
df2.to_csv(csvfile, index=False)
- #print(df2)
def IsUrlValid(url:str)->bool:
@@ -167,19 +165,14 @@ def IsOnionValid(url: str)-> bool:
if len(url.split('.')) > 3:
n_subdomians = len(url.split('.'))
# Checks if there is more than 1 subdomain. "subdomain.url.onion" only
- #print(f"This domain have more than one subdomain. There are {n_subdomians - 1} subdomains")
return False
else:
if len(url) < 62:
- #print("Domain length is less than 62.")
return False
return True
elif pattern.fullmatch(url) is None:
- #print("Domain contains invalid character.")
- #print(url)
return False
else:
- #print("Domain not valid")
return False
except Exception as e:
print(f"Error: {e}")
diff --git a/scripts/utils.py b/scripts/utils.py
index 24db984..e6b6bc4 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -1,158 +1,120 @@
-import os, pwd, re, pandas as pd, requests, shutil
+import re
+import requests
from PIL import Image
-import urllib
-import socks, socket, glob
PURPLE = '\033[35;40m'
BOLD_PURPLE = '\033[35;40;1m'
-ORANGE = '\033[33;40;1m'
RED = '\033[31;40m'
BOLD_RED = '\033[31;40;1m'
RESET = '\033[m'
+
#### Checking Functions to validate that links are legit ####
def CheckUrl(url):
- """
- Checks if URL is actually reachable via Tor
- """
- proxies = {
- 'http': 'socks5h://127.0.0.1:9050',
- 'https': 'socks5h://127.0.0.1:9050'
- }
- try:
- status = requests.get(url,proxies=proxies, timeout=5).status_code
- print('[+]',url,status)
- if status != 502:
- #print(url,"✔️")
- return True
- else:
- #print(url,"❌")
- return False
- except requests.ConnectionError as e:
- #print(url,"❌")
- return False
- except requests.exceptions.ReadTimeout as e:
- #print(url,"❌")
- return False
+ """
+ Checks if URL is actually reachable via Tor
+ """
+ proxies = {
+ 'http': 'socks5h://127.0.0.1:9050',
+ 'https': 'socks5h://127.0.0.1:9050'
+ }
+ try:
+ status = requests.get(url,proxies=proxies, timeout=5).status_code
+ if status != 502:
+ return True
+ else:
+ return False
+ except requests.ConnectionError as e:
+ return False
+ except requests.exceptions.ReadTimeout as e:
+ return False
+
#### PROTECTIONS AGAINST MALICIOUS CSV INPUTS ####
def IsBannerValid(path: str) -> bool:
- """
- Checks if the banner.png file has the correct dimensions (240x60)
- """
- #print('[+] checking image size')
- try:
- im = Image.open(path)
- except Exception as e:
- return False
- #im = Image.open("favicon.png")
- width, height = im.size
- #print('width =',width, 'height=',height)
- if width != 240 or height != 60:
- #print('[-] Banner doesnt have the correct size (240x60)')
- return False
- else:
- #print('[+] Banner has the correct size (240x60)')
- return True
+ """
+ Checks if the banner.png file has the correct dimensions (240x60)
+ """
+ try:
+ im = Image.open(path)
+ except Exception as e:
+ print("ERROR, EXCEPTION")
+ return False
+ width, height = im.size
+ if width != 240 or height != 60:
+ print("INVALID BANNER DIMENSIONS, HEIGHT=",height," WIDTH=",width)
+ return False
+
+ return True
def IsOnionValid(url: str)-> bool:
"""
Checks if the domain(param) is a valid onion domain and return True else False.
"""
- # check if the characters are only [a-zA-Z0-9.] with maximum 128 chars max?
- # check that it is only url.onion or subdomain.url.onion,
- # if OK return True
- #if not : return False
try:
- pattern = re.compile("^[A-Za-z0-9.]+(\.onion)?$")
+ pattern = re.compile("^[A-Za-z0-9.]+(.onion)?$")
url = url.strip().removesuffix('/')
if url.startswith('http://'):
- #print('URL starts with http')
- # Removes the http://
domain = url.split('/')[2]
if pattern.fullmatch(domain) is not None:
if len(domain.split('.')) > 3:
- n_subdomians = len(domain.split('.'))
- # Checks if there is more than 1 subdomain. "subdomain.url.onion" only
- #print(f"This domain have more than one subdomain. There are {n_subdomians} subdomains")
return False
else:
if len(domain) < 62:
- #print("Domain length is less than 62.")
return False
return True
elif pattern.fullmatch(domain) is None:
- #print("Domain contains invalid character.")
- #print(domain)
return False
else:
- #print("Domain not valid")
return False
else:
- #TODO : edit the url to make sure it has http:// at the beginning, in case if it's missing? (problem is that it only returns true or false)
- #print("URL doesn't start http")
+ #TODO : edit the url to make sure it has http:// at the beginning, in case if it's missing? (problem is that it only returns true or false)
if pattern.fullmatch(url) is not None:
if len(url.split('.')) > 3:
- n_subdomians = len(url.split('.'))
- # Checks if there is more than 1 subdomain. "subdomain.url.onion" only
- #print(f"This domain have more than one subdomain. There are {n_subdomians - 1} subdomains")
return False
else:
if len(url) < 62:
- #print("Domain length is less than 62.")
return False
return True
elif pattern.fullmatch(url) is None:
- #print("Domain contains invalid character.")
- #print(url)
return False
else:
- #print("Domain not valid")
return False
except Exception as e:
- print(f"Error: {e}")
+ return False
+
def IsUrlValid(url:str)->bool:
"""
Check if url is valid both dark net end clearnet.
"""
- # check if the characters are only [a-zA-Z0-9.:/] with maximum 128 chars max?
- # check that it is only http(s)://wordA.wordB or http(s)://WordC.WordB.WordC, (onion or not), clearnet is fine too (double check if those are fine!)
- # if OK return True
- #if not : return False
pattern = re.compile("^[A-Za-z0-9:/.-]+$")
url = str(url)
if len(url) < 4:
- #print("Status: Got more than one character or nothing.")
return False
if url.endswith('.onion'):
return IsOnionValid(url)
else:
if not url.__contains__('.'):
- #print("No (DOT) in clearnet url")
return False
if pattern.fullmatch(url) is None:
- #print('Url contains invalid chars')
return False
return True
+
def IsStatusValid(status: str)-> bool:
"""
- Checks if status contains only [v,x,❌,✔️]. Verbose only if False is returned
+ Checks if status contains only ['YES','NO']. Verbose only if False is returned
"""
- pattern = ['y','n','✔️','❌','','nan']
+ pattern = ['YES','NO','✔️','❌','']
+ #pattern = ['YES','NO']
status = str(status)
status.strip()
- #print('[+] STATUS = ',status.splitlines())
- if len(status) > 4:
- #print("Status: Got more than one character or nothing.")
- return False
- elif (status not in pattern):
- #print("Status: Got an invalid character it must be either y, n, ✔️, or ❌ ")
+ if (status not in pattern):
return False
return True
@@ -162,24 +124,15 @@ def IsScoreValid(score:str)->bool:
"""
Check the Score is only "^[0-9.,]+$" with 8 max chars.
"""
- # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max
- #(careful with the ' and , make sure you test if it fucks the csv up or else)
- # if OK return True
- #if not : return False
pattern = re.compile("^[0-9.,]+$")
score = str(score)
score.strip()
- #pattern = ['','nan']
if score in ['','nan']:
- #Score can be empty when initially added
return True
if pattern.fullmatch(score) is None:
- # empty description is fine as it's optional
return False
elif len(score) > 8:
- #print("score is greater than 8 chars")
return False
- # empty score is fine
return True
@@ -187,12 +140,7 @@ def IsDescriptionValid(desc:str)->bool:
"""
Check the categories are only [a-zA-Z0-9.' ] with 256 max chars.
"""
- # check if the characters are only [a-zA-Z0-9.,' ] with maximum 256 chars max
- #(careful with the ' and , make sure you test if it fucks the csv up or else)
- # if OK return True
- #if not : return False
if desc == "":
- # empty description is fine as it's optional
return True
pattern = re.compile("^[A-Za-z0-9-.,' \"]+$")
desc = str(desc)
@@ -202,7 +150,6 @@ def IsDescriptionValid(desc:str)->bool:
if desc == "DEFAULT":
return False
elif len(desc) > 256:
- #print("desc is greater than 256 chars")
return False
return True
@@ -210,18 +157,12 @@ def IsCategoryValid(categories: list)-> bool:
"""
Check the categories are only [a-zA-Z0-9 ] with 64 max chars.
"""
- # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max
- #(careful with the ' and , make sure you test if it fucks the csv up or else)
- # if OK return True
- #if not : return False
pattern = re.compile("^[A-Za-z0-9 ]+$")
for category in categories:
category.strip()
if pattern.fullmatch(category) is None:
- #print('Got an empty list or invalid chars')
return False
elif len(category) > 64:
- #print('Category is too long')
return False
else:
return True
@@ -230,41 +171,31 @@ def IsNameValid(name: str)->bool:
"""
Check the parameter name only contains [a-zA-Z0-9 ] and is 64 chars long.
"""
- # check if the characters are only [a-zA-Z0-9 ] with maximum 64 chars max
- #(careful with the ' and , make sure you test if it fucks the csv up or else)
- # if OK return True
- #if not : return False
+ try:
+ name = str(name)
+ except Exception as e:
+ return False
pattern = re.compile("^[A-Za-z0-9 ]+$")
name = name.strip()
if (pattern.fullmatch(name) is None):
- #print("Got an invalid character or nothing")
return False
elif len(name) > 64:
- #print(f'Got a name length greater than 64. {len(name)}')
return False
return True
-
-#def print_colors(s:str, bold=False, is_error = False, default=False):
-def print_colors(*args, bold=False, is_error=False, default=False, highlight=False):
- """
- Helper function to print with colors
- """
- for s in args:
+def print_colors(s:str=' ', bold:bool=False, is_error:bool = False, default:bool=False):
+ """
+ Helper function to print with colors
+ """
if is_error:
- print(f"{RED}{s}{RESET}",end='')
- elif highlight:
- print(f"{ORANGE}{s}{RESET}",end='')
+ print(f"{RED}{s}{RESET}")
elif bold:
- print(f"{BOLD_PURPLE}{s}{RESET}",end='')
+ print(f"{BOLD_PURPLE}{s}{RESET}")
elif is_error and bold:
- print(f"{BOLD_RED}{s}{RESET}",end='')
+ print(f"{BOLD_RED}{s}{RESET}")
elif default:
- print(f'{s}',end='')
+ print(f'{s}')
else:
- print(f"{PURPLE}{s}{RESET}",end='')
- if s is args[-1]:
- print()
-
+ print(f"{PURPLE}{s}{RESET}")
diff --git a/updaterepo.sh b/updaterepo.sh
new file mode 100755
index 0000000..cd8babd
--- /dev/null
+++ b/updaterepo.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+#warning, you need to have the .gitignore intact to ignore www/participants/*/**
+
+git rm -r --cached .
+git add .
+git commit
+torsocks git push
diff --git a/www/.known_participants b/www/.known_participants
new file mode 100644
index 0000000..574264e
--- /dev/null
+++ b/www/.known_participants
@@ -0,0 +1,3 @@
+lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion
+lantern.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion
+zhd7yf675dav6njgc7yjwke2u5cq7d5qim2s7xwa2ukxfzubrguqmzyd.onion
diff --git a/www/header.php b/www/header.php
index 5188615..c5ef228 100644
--- a/www/header.php
+++ b/www/header.php
@@ -53,7 +53,7 @@ if (!preg_match("~^(?:f|ht)tps?://~i", $data[3])) {
//if ((($sensitive == 1) and ($data[4] == "✔️")) or (($sensitive == 0) and ($data[4] != "✔️")) ){
// ONLY display links if (sensitive equals to 1 and sensitiveCOLUMN equals to V) OR (sensitive equals to 0 and sensitiveCOLUMN is NOT equal to V)
- if (($data[4] != "✔️") or (($sensitive == 1) and ($data[4] == "✔️"))){
+ if (($data[4] != "YES") or (($sensitive == 1) and ($data[4] == "YES"))){
$rowcount++;
@@ -67,7 +67,7 @@ if (!preg_match("~^(?:f|ht)tps?://~i", $data[3])) {
}
echo "| " ; // begin the table cell
- if($data[4] == "✔️"){
+ if($data[4] == "YES"){
echo ' '; // display the link
echo $data[2] . ' | '; // display the link title and close the a href and first cell, open the second cell
echo $data[5] . " | "; // OPTIONAL: display the description column
- echo $data[7] . " | "; // display the status and close the second cell, open the third cell
- echo $data[6] . " | \n"; // display the score and close the third cell
+ echo $data[7] . " "; // display the score and close the second cell, open the third cell
+ if($data[6] == "YES"){
+ echo "✔️" ;
+ }else{
+ echo "❌" ;
+ }
+ echo " | \n"; // display the status and close the third cell
}
//if ($c == 2){
//}
@@ -169,7 +174,7 @@ if (($handle = fopen($csvfile, "r")) !== FALSE) {
}
}
}
- echo "";
+ echo 'Display All Links |';
fclose($handle);
}
//echo "" . $resultcount . " Result(s) found.
";
diff --git a/www/participants/lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/banner.png b/www/participants/lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/banner.png
new file mode 100644
index 0000000..8e7af73
Binary files /dev/null and b/www/participants/lantern.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/banner.png differ
diff --git a/www/participants/lantern.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion/banner.png b/www/participants/lantern.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion/banner.png
new file mode 100644
index 0000000..1e52c73
Binary files /dev/null and b/www/participants/lantern.nowhevi57f4lxxd6db43miewcsgtovakbh6v5f52ci7csc2yjzy5rnid.onion/banner.png differ
diff --git a/www/participants/zhd7yf675dav6njgc7yjwke2u5cq7d5qim2s7xwa2ukxfzubrguqmzyd.onion/banner.png b/www/participants/zhd7yf675dav6njgc7yjwke2u5cq7d5qim2s7xwa2ukxfzubrguqmzyd.onion/banner.png
new file mode 100644
index 0000000..e38120f
Binary files /dev/null and b/www/participants/zhd7yf675dav6njgc7yjwke2u5cq7d5qim2s7xwa2ukxfzubrguqmzyd.onion/banner.png differ