Skip to content
This repository was archived by the owner on Jan 22, 2026. It is now read-only.

Commit 9e667ef

Browse files
andrewandrew
committed
Replace ActiveRecord with Sequel
- 32% fewer gems (41 → 28), 24% smaller lockfile - 50% faster DB operations, 56% faster bulk inserts - Convert all models to Sequel::Model - Update query syntax (includes → eager, joins → eager_graph) - Add disconnect method to clean up Sequel database caches - Clear association caches in refresh_models for test isolation - Update benchmarks for Sequel syntax
1 parent d1f6a38 commit 9e667ef

33 files changed

+506
-462
lines changed

‎Gemfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ source "https://rubygems.org"
44

55
gemspec
66

7-
#gem "ecosystems-bibliothecary", git: "https://github.com/ecosyste-ms/bibliothecary.git", require: "bibliothecary"
7+
gem "ecosystems-bibliothecary", git: "https://github.com/ecosyste-ms/bibliothecary.git", require: "bibliothecary"
8+
# gem "ecosystems-bibliothecary", path: "/Users/andrew/code/ecosystems/bibliothecary", require: "bibliothecary"
89
gem "ostruct"
910

1011
gem "irb"

‎Gemfile.lock‎

Lines changed: 18 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,38 @@
1+
GIT
2+
remote: https://github.com/ecosyste-ms/bibliothecary.git
3+
revision: 827bda17dc8866d3f712d40942760f4d52e23b63
4+
specs:
5+
ecosystems-bibliothecary (15.0.1)
6+
bundler
7+
csv
8+
json (~> 2.8)
9+
ox (>= 2.8.1)
10+
racc
11+
tomlrb (~> 2.0)
12+
113
PATH
214
remote: .
315
specs:
416
git-pkgs (0.5.0)
5-
activerecord (>= 7.0)
617
ecosystems-bibliothecary (~> 15.0)
718
rugged (~> 1.0)
19+
sequel (>= 5.0)
820
sqlite3 (>= 2.0)
921

1022
GEM
1123
remote: https://rubygems.org/
1224
specs:
13-
activemodel (8.1.1)
14-
activesupport (= 8.1.1)
15-
activerecord (8.1.1)
16-
activemodel (= 8.1.1)
17-
activesupport (= 8.1.1)
18-
timeout (>= 0.4.0)
19-
activesupport (8.1.1)
20-
base64
21-
bigdecimal
22-
concurrent-ruby (~> 1.0, >= 1.3.1)
23-
connection_pool (>= 2.2.5)
24-
drb
25-
i18n (>= 1.6, < 2)
26-
json
27-
logger (>= 1.4.2)
28-
minitest (>= 5.1)
29-
securerandom (>= 0.3)
30-
tzinfo (~> 2.0, >= 2.0.5)
31-
uri (>= 0.13.1)
32-
base64 (0.3.0)
3325
benchmark (0.5.0)
3426
bigdecimal (4.0.1)
35-
concurrent-ruby (1.3.6)
36-
connection_pool (3.0.2)
3727
csv (3.3.5)
3828
date (3.5.1)
39-
drb (2.2.3)
40-
ecosystems-bibliothecary (15.0.1)
41-
bundler
42-
csv
43-
json (~> 2.8)
44-
ox (>= 2.8.1)
45-
racc
46-
tomlrb (~> 2.0)
4729
erb (6.0.1)
48-
i18n (1.14.8)
49-
concurrent-ruby (~> 1.0)
5030
io-console (0.8.2)
5131
irb (1.16.0)
5232
pp (>= 0.6.0)
5333
rdoc (>= 4.0.0)
5434
reline (>= 0.4.2)
5535
json (2.18.0)
56-
logger (1.7.0)
5736
minitest (6.0.1)
5837
prism (~> 1.5)
5938
ostruct (0.6.3)
@@ -75,7 +54,8 @@ GEM
7554
reline (0.6.3)
7655
io-console (~> 0.5)
7756
rugged (1.9.0)
78-
securerandom (0.4.1)
57+
sequel (5.100.0)
58+
bigdecimal
7959
sqlite3 (2.9.0-aarch64-linux-gnu)
8060
sqlite3 (2.9.0-aarch64-linux-musl)
8161
sqlite3 (2.9.0-arm-linux-gnu)
@@ -87,12 +67,8 @@ GEM
8767
sqlite3 (2.9.0-x86_64-linux-gnu)
8868
sqlite3 (2.9.0-x86_64-linux-musl)
8969
stringio (3.2.0)
90-
timeout (0.6.0)
9170
tomlrb (2.0.4)
9271
tsort (0.2.0)
93-
tzinfo (2.0.6)
94-
concurrent-ruby (~> 1.0)
95-
uri (1.1.1)
9672

9773
PLATFORMS
9874
aarch64-linux-gnu
@@ -108,32 +84,24 @@ PLATFORMS
10884

10985
DEPENDENCIES
11086
benchmark
87+
ecosystems-bibliothecary!
11188
git-pkgs!
11289
irb
11390
minitest
11491
ostruct
11592
rake
11693

11794
CHECKSUMS
118-
activemodel (8.1.1) sha256=8b7e2496b9e333ced06248c16a43217b950192c98e0fe3aa117eee21501c6fbd
119-
activerecord (8.1.1) sha256=e32c3a03e364fd803498eb4150c21bedc995aa83bc27122a94d480ab1dcb3d17
120-
activesupport (8.1.1) sha256=5e92534e8d0c8b8b5e6b16789c69dbea65c1d7b752269f71a39422e9546cea67
121-
base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b
12295
benchmark (0.5.0) sha256=465df122341aedcb81a2a24b4d3bd19b6c67c1530713fd533f3ff034e419236c
12396
bigdecimal (4.0.1) sha256=8b07d3d065a9f921c80ceaea7c9d4ae596697295b584c296fe599dd0ad01c4a7
124-
concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab
125-
connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
12697
csv (3.3.5) sha256=6e5134ac3383ef728b7f02725d9872934f523cb40b961479f69cf3afa6c8e73f
12798
date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0
128-
drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373
129-
ecosystems-bibliothecary (15.0.1) sha256=e557381cbe1a56931cb68c8a685a33dd9156a7b331a583874b4ac05f94df6f1c
99+
ecosystems-bibliothecary (15.0.1)
130100
erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5
131101
git-pkgs (0.5.0)
132-
i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5
133102
io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
134103
irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806
135104
json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
136-
logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
137105
minitest (6.0.1) sha256=7854c74f48e2e975969062833adc4013f249a4b212f5e7b9d5c040bf838d54bb
138106
ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912
139107
ox (2.14.23) sha256=4a9aedb4d6c78c5ebac1d7287dc7cc6808e14a8831d7adb727438f6a1b461b66
@@ -146,7 +114,7 @@ CHECKSUMS
146114
rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9
147115
reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
148116
rugged (1.9.0) sha256=7faaa912c5888d6e348d20fa31209b6409f1574346b1b80e309dbc7e8d63efac
149-
securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1
117+
sequel (5.100.0) sha256=cb0329b62287a01db68eead46759c14497a3fae01b174e2c41da108a9e9b4a12
150118
sqlite3 (2.9.0-aarch64-linux-gnu) sha256=cfe1e0216f46d7483839719bf827129151e6c680317b99d7b8fc1597a3e13473
151119
sqlite3 (2.9.0-aarch64-linux-musl) sha256=56a35cb2d70779afc2ac191baf2c2148242285ecfed72f9b021218c5c4917913
152120
sqlite3 (2.9.0-arm-linux-gnu) sha256=a19a21504b0d7c8c825fbbf37b358ae316b6bd0d0134c619874060b2eef05435
@@ -158,11 +126,8 @@ CHECKSUMS
158126
sqlite3 (2.9.0-x86_64-linux-gnu) sha256=72fff9bd750070ba3af695511ba5f0e0a2d8a9206f84869640b3e99dfaf3d5a5
159127
sqlite3 (2.9.0-x86_64-linux-musl) sha256=ef716ba7a66d7deb1ccc402ac3a6d7343da17fac862793b7f0be3d2917253c90
160128
stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1
161-
timeout (0.6.0) sha256=6d722ad619f96ee383a0c557ec6eb8c4ecb08af3af62098a0be5057bf00de1af
162129
tomlrb (2.0.4) sha256=262f77947ac3ac9b3366a0a5940ecd238300c553e2e14f22009e2afcd2181b99
163130
tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f
164-
tzinfo (2.0.6) sha256=8daf828cc77bcf7d63b0e3bdb6caa47e2272dcfaf4fbfe46f8c3a9df087a829b
165-
uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6
166131

167132
BUNDLED WITH
168133
4.0.1

‎benchmark/bulk.rb‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@
88
repo_path = ARGV[0] || "/Users/andrew/code/octobox"
99
sample_size = (ARGV[1] || 500).to_i
1010

11-
# In-memory with WAL mode equivalent (journal_mode=memory for in-memory DB)
11+
# In-memory with optimized settings
1212
Git::Pkgs::Database.connect_memory
13-
ActiveRecord::Base.connection.execute("PRAGMA synchronous = OFF")
14-
ActiveRecord::Base.connection.execute("PRAGMA journal_mode = MEMORY")
13+
Git::Pkgs::Database.db.run("PRAGMA synchronous = OFF")
14+
Git::Pkgs::Database.db.run("PRAGMA journal_mode = MEMORY")
1515

1616
repo = Git::Pkgs::Repository.new(repo_path)
1717
analyzer = Git::Pkgs::Analyzer.new(repo)
@@ -111,11 +111,11 @@
111111
# Bulk insert
112112
insert_time = Benchmark.realtime do
113113
# Insert commits
114-
Git::Pkgs::Models::Commit.insert_all(all_commits) if all_commits.any?
114+
Git::Pkgs::Models::Commit.multi_insert(all_commits) if all_commits.any?
115115

116116
# Build SHA -> ID map
117-
commit_ids = Git::Pkgs::Models::Commit.where(sha: all_commits.map { |c| c[:sha] }).pluck(:sha, :id).to_h
118-
manifest_ids = Git::Pkgs::Models::Manifest.pluck(:path, :id).to_h
117+
commit_ids = Git::Pkgs::Models::Commit.where(sha: all_commits.map { |c| c[:sha] }).select_map([:sha, :id]).to_h
118+
manifest_ids = Git::Pkgs::Models::Manifest.select_map([:path, :id]).to_h
119119

120120
# Insert branch_commits with resolved IDs
121121
branch_commit_records = all_branch_commits.map do |bc|
@@ -125,7 +125,7 @@
125125
position: bc[:commit_position]
126126
}
127127
end
128-
Git::Pkgs::Models::BranchCommit.insert_all(branch_commit_records) if branch_commit_records.any?
128+
Git::Pkgs::Models::BranchCommit.multi_insert(branch_commit_records) if branch_commit_records.any?
129129

130130
# Insert changes with resolved IDs
131131
change_records = all_changes.map do |c|
@@ -142,7 +142,7 @@
142142
updated_at: c[:updated_at]
143143
}
144144
end
145-
Git::Pkgs::Models::DependencyChange.insert_all(change_records) if change_records.any?
145+
Git::Pkgs::Models::DependencyChange.multi_insert(change_records) if change_records.any?
146146

147147
# Insert snapshots with resolved IDs
148148
snapshot_records = all_snapshots.map do |s|
@@ -157,7 +157,7 @@
157157
updated_at: s[:updated_at]
158158
}
159159
end
160-
Git::Pkgs::Models::DependencySnapshot.insert_all(snapshot_records) if snapshot_records.any?
160+
Git::Pkgs::Models::DependencySnapshot.multi_insert(snapshot_records) if snapshot_records.any?
161161
end
162162

163163
puts "Insert time: #{insert_time.round(3)}s"

‎benchmark/db.rb‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,10 @@
5252
counts[:commits] += 1
5353

5454
timings[:branch_commit_create] += Benchmark.realtime do
55-
Git::Pkgs::Models::BranchCommit.find_or_create_by(
56-
branch: branch,
57-
commit: commit,
58-
position: position
59-
)
55+
Git::Pkgs::Models::BranchCommit.find_or_create(
56+
branch_id: branch.id,
57+
commit_id: commit.id
58+
) { |bc| bc.position = position }
6059
end
6160
counts[:branch_commits] += 1
6261

@@ -77,7 +76,7 @@
7776
end
7877

7978
timings[:change_create] += Benchmark.realtime do
80-
Git::Pkgs::Models::DependencyChange.create!(
79+
Git::Pkgs::Models::DependencyChange.create(
8180
commit: commit,
8281
manifest: manifest,
8382
name: change[:name],
@@ -95,10 +94,10 @@
9594

9695
snapshot.each do |(manifest_path, name), dep_info|
9796
timings[:snapshot_create] += Benchmark.realtime do
98-
manifest = Git::Pkgs::Models::Manifest.find_by(path: manifest_path)
99-
Git::Pkgs::Models::DependencySnapshot.find_or_create_by(
100-
commit: commit,
101-
manifest: manifest,
97+
manifest = Git::Pkgs::Models::Manifest.first(path: manifest_path)
98+
Git::Pkgs::Models::DependencySnapshot.find_or_create(
99+
commit_id: commit.id,
100+
manifest_id: manifest.id,
102101
name: name
103102
) do |s|
104103
s.ecosystem = dep_info[:ecosystem]

‎git-pkgs.gemspec‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
3232
spec.require_paths = ["lib"]
3333

3434
spec.add_dependency "rugged", "~> 1.0"
35-
spec.add_dependency "activerecord", ">= 7.0"
35+
spec.add_dependency "sequel", ">= 5.0"
3636
spec.add_dependency "sqlite3", ">= 2.0"
3737
spec.add_dependency "ecosystems-bibliothecary", "~> 15.0"
3838
end

‎lib/git/pkgs/commands/blame.rb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,12 @@ def run
1919

2020
# Get current dependencies at the last analyzed commit
2121
branch_name = @options[:branch] || repo.default_branch
22-
branch = Models::Branch.find_by(name: branch_name)
22+
branch = Models::Branch.first(name: branch_name)
2323

2424
error "No analysis found for branch '#{branch_name}'. Run 'git pkgs init' first." unless branch&.last_analyzed_sha
2525

26-
current_commit = Models::Commit.find_by(sha: branch.last_analyzed_sha)
27-
snapshots = current_commit&.dependency_snapshots&.includes(:manifest) || []
26+
current_commit = Models::Commit.first(sha: branch.last_analyzed_sha)
27+
snapshots = current_commit&.dependency_snapshots&.eager(:manifest) || []
2828

2929
if @options[:ecosystem]
3030
snapshots = snapshots.where(ecosystem: @options[:ecosystem])
@@ -41,7 +41,7 @@ def run
4141
names = snapshots.map(&:name).uniq
4242

4343
all_added_changes = Models::DependencyChange
44-
.includes(:commit)
44+
.eager(:commit)
4545
.added
4646
.where(manifest_id: manifest_ids, name: names)
4747
.to_a

‎lib/git/pkgs/commands/branch.rb‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ def add_branch
5050

5151
error "Branch '#{branch_name}' not found. Check 'git branch -a' for available branches." unless repo.branch_exists?(branch_name)
5252

53-
existing = Models::Branch.find_by(name: branch_name)
53+
existing = Models::Branch.first(name: branch_name)
5454
if existing
5555
info "Branch '#{branch_name}' already tracked (#{existing.commits.count} commits)"
5656
info "Use 'git pkgs update' to refresh"
@@ -59,7 +59,7 @@ def add_branch
5959

6060
Database.optimize_for_bulk_writes
6161

62-
branch = Models::Branch.create!(name: branch_name)
62+
branch = Models::Branch.create(name: branch_name)
6363
analyzer = Analyzer.new(repo)
6464

6565
info "Analyzing branch: #{branch_name}"
@@ -100,7 +100,7 @@ def list_branches
100100
puts "Tracked branches:"
101101
branches.each do |branch|
102102
commit_count = branch.commits.count
103-
dep_commits = branch.commits.where(has_dependency_changes: true).count
103+
dep_commits = branch.commits_dataset.where(has_dependency_changes: true).count
104104
last_sha = branch.last_analyzed_sha&.slice(0, 7) || "none"
105105
puts " #{branch.name}: #{commit_count} commits (#{dep_commits} with deps), last: #{last_sha}"
106106
end
@@ -115,13 +115,13 @@ def remove_branch
115115

116116
Database.connect(repo.git_dir)
117117

118-
branch = Models::Branch.find_by(name: branch_name)
118+
branch = Models::Branch.first(name: branch_name)
119119
error "Branch '#{branch_name}' not tracked. Run 'git pkgs branch list' to see tracked branches." unless branch
120120

121121
# Only delete branch_commits, keep shared commits
122122
count = branch.branch_commits.count
123-
branch.branch_commits.delete_all
124-
branch.destroy
123+
branch.branch_commits_dataset.delete
124+
branch.delete
125125

126126
info "Removed branch '#{branch_name}' (#{count} branch-commit links)"
127127
end
@@ -143,28 +143,27 @@ def bulk_process_commits(commits, branch, analyzer, total, repo)
143143
flush = lambda do
144144
return if pending_commits.empty?
145145

146-
ActiveRecord::Base.transaction do
147-
# Use upsert for commits since they may already exist from other branches
146+
Database.db.transaction do
147+
# Use insert with on_conflict for commits since they may already exist from other branches
148148
if pending_commits.any?
149-
Models::Commit.upsert_all(
150-
pending_commits,
151-
unique_by: :sha
152-
)
149+
Models::Commit.dataset
150+
.insert_conflict(target: :sha, update: { has_dependency_changes: Sequel[:excluded][:has_dependency_changes] })
151+
.multi_insert(pending_commits)
153152
end
154153

155154
commit_ids = Models::Commit
156155
.where(sha: pending_commits.map { |c| c[:sha] })
157-
.pluck(:sha, :id).to_h
156+
.select_hash(:sha, :id)
158157

159158
if pending_branch_commits.any?
160159
branch_commit_records = pending_branch_commits.map do |bc|
161160
{ branch_id: bc[:branch_id], commit_id: commit_ids[bc[:sha]], position: bc[:position] }
162161
end
163-
Models::BranchCommit.insert_all(branch_commit_records)
162+
Models::BranchCommit.dataset.multi_insert(branch_commit_records)
164163
end
165164

166165
if pending_changes.any?
167-
manifest_ids = Models::Manifest.pluck(:path, :id).to_h
166+
manifest_ids = Models::Manifest.select_hash(:path, :id)
168167
change_records = pending_changes.map do |c|
169168
{
170169
commit_id: commit_ids[c[:sha]],
@@ -179,11 +178,11 @@ def bulk_process_commits(commits, branch, analyzer, total, repo)
179178
updated_at: now
180179
}
181180
end
182-
Models::DependencyChange.insert_all(change_records)
181+
Models::DependencyChange.dataset.multi_insert(change_records)
183182
end
184183

185184
if pending_snapshots.any?
186-
manifest_ids ||= Models::Manifest.pluck(:path, :id).to_h
185+
manifest_ids ||= Models::Manifest.select_hash(:path, :id)
187186
snapshot_records = pending_snapshots.map do |s|
188187
{
189188
commit_id: commit_ids[s[:sha]],
@@ -196,7 +195,7 @@ def bulk_process_commits(commits, branch, analyzer, total, repo)
196195
updated_at: now
197196
}
198197
end
199-
Models::DependencySnapshot.insert_all(snapshot_records)
198+
Models::DependencySnapshot.dataset.multi_insert(snapshot_records)
200199
end
201200
end
202201

0 commit comments

Comments
 (0)