Project

General

Profile

1
import boto
2
import boto.s3.connection
3
import sys
4
from pymongo import MongoClient
5
import os.path
6
import time
7

    
8

    
9

    
10
def file_exist(bucket, file_path):
11
	c =bucket.get_key(file_path)
12
	if c is not None and c.size > 0:
13
		return True
14
	return False
15

    
16
def exportItemForMongoCollection(obsId, db, bucket, log_file):	
17
	destination_collection =db[ 's3_'+obsId[:36]]
18
	source_collection = db[ obsId[:36]]
19
	print 'Start to exporting objectStore :%s '%obsId
20
	i = 0
21
	last_percentage = 0
22
	total = source_collection.estimated_document_count()
23
	for item in source_collection.find(no_cursor_timeout=True):
24
		fs_path = item['fsPath']
25
		objectId = item['id']
26
		if os.path.isfile(fs_path):
27
			i += 1
28
			if not file_exist(bucket, '%s/%s'%(obsId,objectId)):
29
				key = bucket.new_key('%s/%s'%(obsId,objectId))
30
				try:
31
					key.set_contents_from_filename(fs_path)	
32
				except Exception as e:
33
					time.sleep(10)
34
					key.set_contents_from_filename(fs_path)
35
				
36
				item.pop('_id', None)
37
				item.pop('fsPath')
38
				item['uri'] = 's3://%s/%s'%(bucket.name, key.name)
39
				destination_collection.insert_one(item)
40
			if i % 1000 == 0:
41
				print "Exported %i/%i"%(i, total)
42
		else:
43
			log_file.writeline('Missing file for objectStoreid: %s ObjectId:%s path: %s'%(obsId, objectId, fs_path))
44
	
45

    
46
def start_import(metadataCollection, bucket, log_file):
47
	client = MongoClient()
48
	db = client['objectStore']
49
	metadataCollection = db[metadataCollection]
50
	for item in metadataCollection.find(no_cursor_timeout=True):
51
		obsId = item['obsId']
52
		exportItemForMongoCollection(obsId, db, bucket, log_file)
53

    
54

    
55

    
56

    
57

    
58
if __name__=='__main__':
59
	args = sys.argv
60
	if not len(args) is not 3 :
61
		print "Error on applying script usage: python scriptName.py s3cfgPath objectstoreBucket"
62
	f = open(args[1])
63
	props = {}	
64
	for line in f:
65
		d =line.split('=')
66
		if len(d) == 2:
67
			props[d[0].strip()] = d[1].strip()
68

    
69
	bname = args[2]
70
	conn = boto.connect_s3(aws_access_key_id = props['access_key'], aws_secret_access_key = props['secret_key'],  host = props['host_base'], calling_format = boto.s3.connection.OrdinaryCallingFormat())
71
	bucket = conn.get_bucket(bname, validate=True)
72
	log_file = open('s3_migration.log', 'w')
73
	start_import('metadataObjectStore',bucket, log_file)
74
	log_file.close()
    (1-1/1)