Project

General

Profile

« Previous | Next » 

Revision 54595

Added by Sandro La Bruzzo over 5 years ago

added skip store functionality

View differences:

s3_migration.py
19 19 
	source_collection = db[ obsId[:36]]
20 20 
	print 'Start to exporting objectStore :%s '%obsId
21 21 
	i = 0
22 
	skip = 0
22 23 
	last_percentage = 0
23 24 
	total = source_collection.estimated_document_count()
24 25 
	for item in source_collection.find(no_cursor_timeout=True):
......
35 36 
					except Exception as e:
36 37 
						time.sleep(10)
37 38 
						key.set_contents_from_filename(fs_path)
38
39 39 
					item.pop('_id', None)
40 40 
					item.pop('fsPath')
41 41 
					item['uri'] = 's3://%s/%s'%(bucket.name, key.name)
42 42 
					destination_collection.insert_one(item)
43 
				if i % 1000 == 0:
44 
					print "Exported %i/%i"%(i, total)
43 
				if i % 100 == 0:
44 
					print "Exported %i/%i   (skipped)%i "%(i, total, skip)
45 45 
			else:
46 46 
				log_file.writeline('Missing file for objectStoreid: %s ObjectId:%s path: %s'%(obsId, objectId, fs_path))
47 
		else:
48 
			skip += 1
47 49
48 50 

  		




  49
  51
  
    
def start_import(metadataCollection, bucket, log_file, done_file, skip_store):


  		












Also available in:  Unified diff