From fcbcd27a4b2b3bd21a85b3d190f72adddb5b3fa5 Mon Sep 17 00:00:00 2001 From: Stephen D <webmaster@scd31.com> Date: Thu, 21 Nov 2024 21:29:54 -0400 Subject: [PATCH] improve edge insertion speed, I hope --- src/bin/import.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/bin/import.rs b/src/bin/import.rs index e0f1b81..8645379 100644 --- a/src/bin/import.rs +++ b/src/bin/import.rs @@ -238,7 +238,27 @@ async fn get_edges( }) .unzip(); - query!("INSERT INTO edges(from_id, to_id) SELECT UNNEST($1::INT[]), UNNEST($2::INT[]) ON CONFLICT DO NOTHING", &from_ids, &to_ids).execute(&p).await?; + query!( + r#" +WITH unique_edges AS ( + SELECT UNNEST($1::INT[]) AS from_id, UNNEST($2::INT[]) AS to_id +) +INSERT INTO edges(from_id, to_id) +SELECT ue.from_id, ue.to_id +FROM unique_edges ue +WHERE NOT EXISTS ( + SELECT 1 + FROM edges e + WHERE e.from_id = ue.from_id + AND e.to_id = ue.to_id +) +ON CONFLICT DO NOTHING +"#, + &from_ids, + &to_ids + ) + .execute(&p) + .await?; statsd.count("edges.inserts", from_ids.len() as f64, &None); -- GitLab