From fcbcd27a4b2b3bd21a85b3d190f72adddb5b3fa5 Mon Sep 17 00:00:00 2001
From: Stephen D <webmaster@scd31.com>
Date: Thu, 21 Nov 2024 21:29:54 -0400
Subject: [PATCH] improve edge insertion speed, I hope

---
 src/bin/import.rs | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/bin/import.rs b/src/bin/import.rs
index e0f1b81..8645379 100644
--- a/src/bin/import.rs
+++ b/src/bin/import.rs
@@ -238,7 +238,27 @@ async fn get_edges(
                         })
                         .unzip();
 
-                    query!("INSERT INTO edges(from_id, to_id) SELECT UNNEST($1::INT[]), UNNEST($2::INT[]) ON CONFLICT DO NOTHING", &from_ids, &to_ids).execute(&p).await?;
+                    query!(
+                        r#"
+WITH unique_edges AS (
+    SELECT UNNEST($1::INT[]) AS from_id, UNNEST($2::INT[]) AS to_id
+)
+INSERT INTO edges(from_id, to_id)
+SELECT ue.from_id, ue.to_id
+FROM unique_edges ue
+WHERE NOT EXISTS (
+    SELECT 1
+    FROM edges e
+    WHERE e.from_id = ue.from_id
+        AND e.to_id = ue.to_id
+)
+ON CONFLICT DO NOTHING
+"#,
+                        &from_ids,
+                        &to_ids
+                    )
+                    .execute(&p)
+                    .await?;
 
                     statsd.count("edges.inserts", from_ids.len() as f64, &None);
 
-- 
GitLab