diff --git a/.github/workflows/check-duplicates.yml b/.github/workflows/check-duplicates.yml
new file mode 100644
index 0000000..2a36b31
--- /dev/null
+++ b/.github/workflows/check-duplicates.yml
@@ -0,0 +1,22 @@
+name: Check Duplicates
+
+on:
+ pull_request:
+ branches:
+ - main
+ push:
+ branches:
+ - main
+
+jobs:
+ check-duplicates:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Check Out Code
+ uses: actions/checkout@v2
+
+ - name: Run Check Duplicates Script
+ run: |
+ chmod +x scripts/check-dups.sh
+ ./scripts/check-dups.sh
+ working-directory: ${{ github.workspace }}
diff --git a/README.md b/README.md
index c34a5e0..a0afce8 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ A curated list of awesome search engines useful during Penetration testing, Vuln
Vulnerabilities •
Exploits •
Attack surface •
- Code •
+ Code •
Mail addresses •
Domains •
URLs •
@@ -42,6 +42,7 @@ A curated list of awesome search engines useful during Penetration testing, Vuln
- [You](https://you.com/)
- [SearXNG](https://searx.be/?q=)
- [EXALead](http://www.exalead.com/search/web/)
+- [DuckDuckGo](https://duckduckgo.com/)
- [Swisscows](https://swisscows.com/en)
### Servers
@@ -128,7 +129,7 @@ A curated list of awesome search engines useful during Penetration testing, Vuln
- [Deepinfo](https://www.deepinfo.com/) - Empower your security with the most comprehensive Internet data
- [Detectify](https://detectify.com/) - Complete External Attack Surface Management
-### Code Search Engines
+### Code
- [GitHub Code Search](https://github.com/search?type=code) - Search globally across all of GitHub, or scope your search to a particular repository or organization
- [GitLab Code Search](https://gitlab.com/) - Advanced search for faster, more efficient search across the entire GitLab instance
@@ -194,7 +195,6 @@ A curated list of awesome search engines useful during Penetration testing, Vuln
- [MoonSearch](http://moonsearch.com/) - Backlinks checker & SEO Report
- [sitereport.netcraft.com](https://sitereport.netcraft.com/) - Find out the infrastructure and technologies used by any site
- [SynapsInt](https://synapsint.com/) - The unified OSINT research tool
-- [spyonweb.com](https://spyonweb.com/) - Find out related websites
- [statscrop.com](https://www.statscrop.com/) - Millions of amazing websites across the web are being analyzed with StatsCrop
- [securityheaders.com](https://securityheaders.com/) - Scan your site now
- [visualsitemapper.com](http://www.visualsitemapper.com/) - Create a visual map of your site
@@ -375,6 +375,9 @@ These can be useful for osint and social engineering.
- [Infringement Report](https://infringement.report/) - The web's best image copyright infringement search tool
- [Tineye](https://tineye.com/) - Image search and recognition company
- [Flickr](https://flickr.com/search/) - Home to tens of billions of photos and 2 million groups
+- [Sogou](https://pic.sogou.com/) - Chinese technology company that offers a search engine
+- [Jimpl](https://jimpl.com/) - Online photo metadata and EXIF data viewer
+- [Same Energy](https://same.energy/) - Find beautiful images
### Threat Intelligence
diff --git a/scripts/check-dups.sh b/scripts/check-dups.sh
index bab7ff9..6e50ca8 100755
--- a/scripts/check-dups.sh
+++ b/scripts/check-dups.sh
@@ -14,15 +14,23 @@ then
readme="../README.md"
fi
-links=$(cat $readme | egrep "\- \[" | wc -l)
+# Function to extract links from a section and check for duplicates
+check_section() {
+ section=$1
+ section_content=$(awk -v section="$section" '/^### / {p=0} {if(p)print} /^### '"$section"'/ {p=1}' "$readme")
+ duplicate_links=$(echo "$section_content" | grep -oP '\[.*?\]\(\K[^)]+' | sort | uniq -d)
+ if [[ -n $duplicate_links ]]; then
+ echo "[ ERR ] DUPLICATE LINKS FOUND"
+ echo "$duplicate_links"
+ exit 1
+ fi
+}
-uniqlinks=$(cat $readme | egrep "\- \[" | uniq | wc -l)
+# Get all unique section headings from the README file and handle spaces and slashes
+sections=$(grep '^### ' "$readme" | sed 's/^### //' | sed 's/[\/&]/\\&/g')
-if [[ $links -eq $uniqlinks ]];
-then
- echo "[ OK! ] NO DUPLICATES FOUND."
- echo "$links links in README."
-else
- echo "[ ERR ] DUPLICATES FOUND!"
- cat $readme | egrep "\- \[" | uniq -c | egrep -iv "1 - ["
-fi
\ No newline at end of file
+# Call the function for each section
+for section in $sections; do
+ check_section "$section"
+done
+echo "[ OK! ] NO DUPLICATES FOUND."
\ No newline at end of file