From fcbcd27a4b2b3bd21a85b3d190f72adddb5b3fa5 Mon Sep 17 00:00:00 2001
From: Stephen D <webmaster@scd31.com>
Date: Thu, 21 Nov 2024 21:29:54 -0400
Subject: [PATCH] improve edge insertion speed, I hope
---
src/bin/import.rs | 22 +++++++++++++++++++++-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/src/bin/import.rs b/src/bin/import.rs
index e0f1b81..8645379 100644
--- a/src/bin/import.rs
+++ b/src/bin/import.rs
@@ -238,7 +238,27 @@ async fn get_edges(
})
.unzip();
- query!("INSERT INTO edges(from_id, to_id) SELECT UNNEST($1::INT[]), UNNEST($2::INT[]) ON CONFLICT DO NOTHING", &from_ids, &to_ids).execute(&p).await?;
+ query!(
+ r#"
+WITH unique_edges AS (
+ SELECT UNNEST($1::INT[]) AS from_id, UNNEST($2::INT[]) AS to_id
+)
+INSERT INTO edges(from_id, to_id)
+SELECT ue.from_id, ue.to_id
+FROM unique_edges ue
+WHERE NOT EXISTS (
+ SELECT 1
+ FROM edges e
+ WHERE e.from_id = ue.from_id
+ AND e.to_id = ue.to_id
+)
+ON CONFLICT DO NOTHING
+"#,
+ &from_ids,
+ &to_ids
+ )
+ .execute(&p)
+ .await?;
statsd.count("edges.inserts", from_ids.len() as f64, &None);
--
GitLab