#!/bin/sh

SOURCE_DIR="/usr/share/nginx/html/lists"
OUTPUT_FILE="$SOURCE_DIR/all.txt"
TEMP_FILE=$(mktemp)

find "$SOURCE_DIR" -maxdepth 1 -name "*.txt" ! -name "all.txt" -exec awk 1 {} + > "$TEMP_FILE"

sed 's/#.*//g' "$TEMP_FILE" | sed 's/\r//' | grep -v '^$' | sort -u > "$OUTPUT_FILE"

# Cleanup
rm "$TEMP_FILE"