feat: remove duplicates

vwh · Sep 1, 2024 · 267ff66 · 267ff66
1 parent 5afd2f9
commit 267ff66
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 6 deletions.
diff --git a/src/App.tsx b/src/App.tsx
@@ -5,12 +5,13 @@ import Results from "./components/results";
 
 export default function App() {
   const [results, setResults] = useState<string[]>([]);
+  const [lengths, setLengths] = useState({ oldLength: 0, newLength: 0 });
 
   return (
     <main className="container mx-auto h-screen p-4">
       <div className="flex h-full flex-col gap-4 pb-7">
-        <Resources setResults={setResults} />
-        <Results results={results} />
+        <Resources setResults={setResults} setLengths={setLengths} />
+        <Results results={results} lengths={lengths} />
       </div>
     </main>
   );

diff --git a/src/components/resources.tsx b/src/components/resources.tsx
@@ -22,9 +22,15 @@ const RESOURCE_TYPES: { name: ProxyType; label: string }[] = [
 
 interface ResourcesProps {
   setResults: React.Dispatch<React.SetStateAction<string[]>>;
+  setLengths: React.Dispatch<
+    React.SetStateAction<{
+      oldLength: number;
+      newLength: number;
+    }>
+  >;
 }
 
-export default function Resources({ setResults }: ResourcesProps) {
+export default function Resources({ setResults, setLengths }: ResourcesProps) {
   const [isSaving, setIsSaving] = useState(false);
   const [isCrawling, setIsCrawling] = useState(false);
   const [selectedResource, setSelectedResource] = useState<ProxyType>("http/s");
@@ -57,22 +63,36 @@ export default function Resources({ setResults }: ResourcesProps) {
   const handleStartCrawling = useCallback(async () => {
     setIsCrawling(true);
     setResults([]);
+    setLengths({ oldLength: 0, newLength: 0 });
     try {
       const neededResources = Array.from(
         // remove duplicates from resources
         new Set(Object.values(resources[selectedResource]).flat())
       );
-      neededResources.forEach(async (url) => {
+      const crawlPromises = neededResources.map(async (url) => {
         const results = await scrap(url);
         const flattenedResults = results
           .flat()
           .filter((result): result is string => typeof result === "string");
+        setLengths((prev) => ({
+          ...prev,
+          oldLength: prev.oldLength + flattenedResults.length
+        }));
         setResults((prev) => [...prev, ...flattenedResults]);
       });
-      // remove duplicates from results
+      await Promise.all(crawlPromises);
     } catch (error) {
       console.error("Error during crawling:", error);
     } finally {
+      // remove duplicates from results
+      setResults((prev) => {
+        const uniqueResults = Array.from(new Set(prev));
+        setLengths((prevLengths) => ({
+          ...prevLengths,
+          newLength: uniqueResults.length
+        }));
+        return uniqueResults;
+      });
       setIsCrawling(false);
     }
   }, [resources, selectedResource, setResults]);

diff --git a/src/components/results.tsx b/src/components/results.tsx
@@ -25,9 +25,13 @@ const SAVE_OPTIONS: {
 
 interface ResultProps {
   results: string[];
+  lengths: {
+    oldLength: number;
+    newLength: number;
+  };
 }
 
-export default function Results({ results }: ResultProps) {
+export default function Results({ results, lengths }: ResultProps) {
   const [isCopying, setIsCopying] = useState(false);
 
   const handleExport = useCallback(
@@ -92,6 +96,12 @@ export default function Results({ results }: ResultProps) {
       {results.length > 0 && (
         <p className="absolute left-0 top-0 z-50 w-full rounded-t bg-primary p-2 text-sm text-background">
           {results.length} Results
+          {lengths.newLength > 0 && (
+            <span>
+              , Remove {Math.abs(lengths.oldLength - lengths.newLength)}{" "}
+              duplicates
+            </span>
+          )}
         </p>
       )}
       <Textarea