pad.pub0.org/bin/dirty-db-cleaner.py

49 lines
1.5 KiB
Python
Raw Normal View History

2019-01-22 07:43:49 +00:00
#!/usr/bin/env PYTHONUNBUFFERED=1 python
2014-12-01 22:39:42 +00:00
#
# Created by Bjarni R. Einarsson, placed in the public domain. Go wild!
#
import json
import os
import sys
try:
dirtydb_input = sys.argv[1]
dirtydb_output = '%s.new' % dirtydb_input
assert(os.path.exists(dirtydb_input))
assert(not os.path.exists(dirtydb_output))
except:
2019-01-22 07:43:49 +00:00
print()
print('Usage: %s /path/to/dirty.db' % sys.argv[0])
print()
print('Note: Will create a file named dirty.db.new in the same folder,')
print(' please make sure permissions are OK and a file by that')
print(' name does not exist already. This script works by omitting')
print(' duplicate lines from the dirty.db file, keeping only the')
print(' last (latest) instance. No revision data should be lost,')
print(' but be careful, make backups. If it breaks you get to keep')
print(' both pieces!')
print()
2014-12-01 22:39:42 +00:00
sys.exit(1)
dirtydb = {}
lines = 0
with open(dirtydb_input, 'r') as fd:
2019-01-22 07:43:49 +00:00
print('Reading %s' % dirtydb_input)
2014-12-01 22:39:42 +00:00
for line in fd:
lines += 1
try:
data = json.loads(line)
dirtydb[data['key']] = line
except:
print("Skipping invalid JSON!")
2014-12-01 22:39:42 +00:00
if lines % 10000 == 0:
sys.stderr.write('.')
2019-01-22 07:43:49 +00:00
print()
print('OK, found %d unique keys in %d lines' % (len(dirtydb), lines))
2014-12-01 22:39:42 +00:00
with open(dirtydb_output, 'w') as fd:
2019-01-22 07:43:49 +00:00
for data in list(dirtydb.values()):
2014-12-01 22:39:42 +00:00
fd.write(data)
2019-01-22 07:43:49 +00:00
print('Wrote data to %s. All done!' % dirtydb_output)