Project

General

Profile

« Previous | Next » 

Revision 58269

Added by Michele Artini about 4 years ago

profiles for migration of entities from mongo/postgres to hadoop (DHP)

View differences:

modules/dnet-openaireplus-profiles/trunk/src/main/resources/eu/dnetlib/test/profiles/HadoopJobConfigurationDSResources/HadoopJobConfigurationDSResourceType/dhp_migration_all_steps.xml
1 
<RESOURCE_PROFILE>
2 
    <HEADER>
3 
        <RESOURCE_IDENTIFIER value="46d02888-2827-438e-8f74-7f57a87b6d38_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ=="/>
4 
        <RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType"/>
5 
        <RESOURCE_KIND value="HadoopJobConfigurationDSResources"/>
6 
        <RESOURCE_URI value=""/>
7 
        <DATE_OF_CREATION value="2001-12-31T12:00:00"/>
8 
    </HEADER>
9 
    <BODY>
10 
        <HADOOP_JOB name="dhpMigrationAllSteps" type="oozie">
11 
            <DESCRIPTION>Job for migration of entities from mongo and postgres to the new Hadoop (all steps)</DESCRIPTION>
12 
            <STATIC_CONFIGURATION>
13 
				<!-- Cluster wide -->
14 
                <PROPERTY key="queueName" value="default"/>
15 
                <PROPERTY key="user.name" value="dnet"/> <!-- username = sqoop?? -->
16 
				<PROPERTY key="oozie.wf.application.path" value="/user/michele.artini/migration/wfs/regular_all_steps/oozie_app"/><!-- edit this property! -->
17 
				<PROPERTY key="hbase_dump_distcp_memory_mb" value="6144"/>
18 
				<PROPERTY key="hbase_dump_distcp_num_maps" value="1"/>
19 
            </STATIC_CONFIGURATION>
20 
            <JOB_INTERFACE>
21 
                <PARAM name="postgresURL" required="true" description="postgres URL"/>
22 
                <PARAM name="postgresUser" required="true" description="postgres user"/>
23 
                <PARAM name="postgresPassword" required="true" description="postgres password"/>
24 
                <PARAM name="mongoURL" required="true" description="mongo url"/>
25 
                <PARAM name="mongoDb" required="true" description="mongo database"/>
26 
                <PARAM name="migrationPathStep1" required="true" description="migration path - step 1"/>
27 
                <PARAM name="migrationPathStep2" required="true" description="migration path - step 2"/>
28 
                <PARAM name="migrationPathStep3" required="true" description="migration path - step 3"/>
29 
            </JOB_INTERFACE>
30 
        </HADOOP_JOB>
31 
        <STATUS>
32 
            <LAST_SUBMISSION_DATE value="2014-11-14T19:57:25+00:00"/>
33 
            <RUNNING_INSTANCES value="0"/>
34 
            <CUMULATIVE_RUN value="75"/>
35 
        </STATUS>
36 
        <SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>
37 
    </BODY>
38 
</RESOURCE_PROFILE>
modules/dnet-openaireplus-profiles/trunk/src/main/resources/eu/dnetlib/test/profiles/HadoopJobConfigurationDSResources/HadoopJobConfigurationDSResourceType/dhp_migration_claims.xml
1 
<RESOURCE_PROFILE>
2 
    <HEADER>
3 
        <RESOURCE_IDENTIFIER value="b7bc3c73-6bee-4a4f-a869-f0057d553dba_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ=="/>
4 
        <RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType"/>
5 
        <RESOURCE_KIND value="HadoopJobConfigurationDSResources"/>
6 
        <RESOURCE_URI value=""/>
7 
        <DATE_OF_CREATION value="2001-12-31T12:00:00"/>
8 
    </HEADER>
9 
    <BODY>
10 
        <HADOOP_JOB name="dhpMigrationClaims" type="oozie">
11 
            <DESCRIPTION>Job for migration of claims from mongo and postgres to the new Hadoop</DESCRIPTION>
12 
            <STATIC_CONFIGURATION>
13 
				<!-- Cluster wide -->
14 
                <PROPERTY key="queueName" value="default"/>
15 
                <PROPERTY key="user.name" value="dnet"/> <!-- username = sqoop?? -->
16 
				<PROPERTY key="oozie.wf.application.path" value="/user/michele.artini/migration/wfs/claims/oozie_app"/><!-- edit this property! -->
17 
				<PROPERTY key="hbase_dump_distcp_memory_mb" value="6144"/>
18 
				<PROPERTY key="hbase_dump_distcp_num_maps" value="1"/>
19 
            </STATIC_CONFIGURATION>
20 
            <JOB_INTERFACE>
21 
                <PARAM name="postgresURL" required="true" description="postgres URL"/>
22 
                <PARAM name="postgresUser" required="true" description="postgres user"/>
23 
                <PARAM name="postgresPassword" required="true" description="postgres password"/>
24 
                <PARAM name="mongoURL" required="true" description="mongo url"/>
25 
                <PARAM name="mongoDb" required="true" description="mongo database"/>
26 
                <PARAM name="migrationClaimsPathStep1" required="true" description="migration claims path - step 1"/>
27 
                <PARAM name="migrationClaimsPathStep2" required="true" description="migration claims path - step 2"/>
28 
                <PARAM name="migrationClaimsPathStep3" required="true" description="migration claims path - step 3"/>
29 
            </JOB_INTERFACE>
30 
        </HADOOP_JOB>
31 
        <STATUS>
32 
            <LAST_SUBMISSION_DATE value="2014-11-14T19:57:25+00:00"/>
33 
            <RUNNING_INSTANCES value="0"/>
34 
            <CUMULATIVE_RUN value="75"/>
35 
        </STATUS>
36 
        <SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>
37 
    </BODY>
38 
</RESOURCE_PROFILE>
modules/dnet-openaireplus-profiles/trunk/src/main/resources/eu/dnetlib/test/profiles/HadoopJobConfigurationDSResources/HadoopJobConfigurationDSResourceType/dhp_migration_step1.xml
1 
<RESOURCE_PROFILE>
2 
    <HEADER>
3 
        <RESOURCE_IDENTIFIER value="f099602d-2d47-4b31-8b3f-21bc659908c8_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ=="/>
4 
        <RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType"/>
5 
        <RESOURCE_KIND value="HadoopJobConfigurationDSResources"/>
6 
        <RESOURCE_URI value=""/>
7 
        <DATE_OF_CREATION value="2001-12-31T12:00:00"/>
8 
    </HEADER>
9 
    <BODY>
10 
        <HADOOP_JOB name="dhpMigrationStep1" type="oozie">
11 
            <DESCRIPTION>Job for migration of entities from mongo and postgres to the new Hadoop (step 1)</DESCRIPTION>
12 
            <STATIC_CONFIGURATION>
13 
				<!-- Cluster wide -->
14 
                <PROPERTY key="queueName" value="default"/>
15 
                <PROPERTY key="user.name" value="dnet"/> <!-- username = sqoop?? -->
16 
				<PROPERTY key="oozie.wf.application.path" value="/user/michele.artini/migration/wfs/regular_step1/oozie_app"/><!-- edit this property! -->
17 
				<PROPERTY key="hbase_dump_distcp_memory_mb" value="6144"/>
18 
				<PROPERTY key="hbase_dump_distcp_num_maps" value="1"/>
19 
            </STATIC_CONFIGURATION>
20 
            <JOB_INTERFACE>
21 
                <PARAM name="postgresURL" required="true" description="postgres URL"/>
22 
                <PARAM name="postgresUser" required="true" description="postgres user"/>
23 
                <PARAM name="postgresPassword" required="true" description="postgres password"/>
24 
                <PARAM name="mongoURL" required="true" description="mongo url"/>
25 
                <PARAM name="mongoDb" required="true" description="mongo database"/>
26 
                <PARAM name="migrationPathStep1" required="true" description="migration path - step 1 (target)"/>
27 
            </JOB_INTERFACE>
28 
        </HADOOP_JOB>
29 
        <STATUS>
30 
            <LAST_SUBMISSION_DATE value="2014-11-14T19:57:25+00:00"/>
31 
            <RUNNING_INSTANCES value="0"/>
32 
            <CUMULATIVE_RUN value="75"/>
33 
        </STATUS>
34 
        <SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>
35 
    </BODY>
36 
</RESOURCE_PROFILE>
modules/dnet-openaireplus-profiles/trunk/src/main/resources/eu/dnetlib/test/profiles/HadoopJobConfigurationDSResources/HadoopJobConfigurationDSResourceType/dhp_migration_step2.xml
1 
<RESOURCE_PROFILE>
2 
    <HEADER>
3 
        <RESOURCE_IDENTIFIER value="7365dce7-d0e8-4a73-a145-8e37e696b04c_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ=="/>
4 
        <RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType"/>
5 
        <RESOURCE_KIND value="HadoopJobConfigurationDSResources"/>
6 
        <RESOURCE_URI value=""/>
7 
        <DATE_OF_CREATION value="2001-12-31T12:00:00"/>
8 
    </HEADER>
9 
    <BODY>
10 
        <HADOOP_JOB name="dhpMigrationStep2" type="oozie">
11 
            <DESCRIPTION>Job for migration of entities from mongo and postgres to the new Hadoop (step 2)</DESCRIPTION>
12 
            <STATIC_CONFIGURATION>
13 
				<!-- Cluster wide -->
14 
                <PROPERTY key="queueName" value="default"/>
15 
                <PROPERTY key="user.name" value="dnet"/> <!-- username = sqoop?? -->
16 
				<PROPERTY key="oozie.wf.application.path" value="/user/michele.artini/migration/wfs/regular_step2/oozie_app"/><!-- edit this property! -->
17 
				<PROPERTY key="hbase_dump_distcp_memory_mb" value="6144"/>
18 
				<PROPERTY key="hbase_dump_distcp_num_maps" value="1"/>
19 
            </STATIC_CONFIGURATION>
20 
            <JOB_INTERFACE>
21 
                <PARAM name="postgresURL" required="true" description="postgres URL"/>
22 
                <PARAM name="postgresUser" required="true" description="postgres user"/>
23 
                <PARAM name="postgresPassword" required="true" description="postgres password"/>
24 
                <PARAM name="migrationPathStep1" required="true" description="migration path - step 1 (source)"/>
25 
                <PARAM name="migrationPathStep2" required="true" description="migration path - step 2 (target)"/>
26 
            </JOB_INTERFACE>
27 
        </HADOOP_JOB>
28 
        <STATUS>
29 
            <LAST_SUBMISSION_DATE value="2014-11-14T19:57:25+00:00"/>
30 
            <RUNNING_INSTANCES value="0"/>
31 
            <CUMULATIVE_RUN value="75"/>
32 
        </STATUS>
33 
        <SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>
34 
    </BODY>
35 
</RESOURCE_PROFILE>
modules/dnet-openaireplus-profiles/trunk/src/main/resources/eu/dnetlib/test/profiles/HadoopJobConfigurationDSResources/HadoopJobConfigurationDSResourceType/dhp_migration_step3.xml
1 
<RESOURCE_PROFILE>
2 
    <HEADER>
3 
        <RESOURCE_IDENTIFIER value="eb22b151-ae6f-48e8-8a85-b96b2856efed_SGFkb29wSm9iQ29uZmlndXJhdGlvbkRTUmVzb3VyY2VzL0hhZG9vcEpvYkNvbmZpZ3VyYXRpb25EU1Jlc291cmNlVHlwZQ=="/>
4 
        <RESOURCE_TYPE value="HadoopJobConfigurationDSResourceType"/>
5 
        <RESOURCE_KIND value="HadoopJobConfigurationDSResources"/>
6 
        <RESOURCE_URI value=""/>
7 
        <DATE_OF_CREATION value="2001-12-31T12:00:00"/>
8 
    </HEADER>
9 
    <BODY>
10 
        <HADOOP_JOB name="dhpMigrationStep3" type="oozie">
11 
            <DESCRIPTION>Job for migration of entities from mongo and postgres to the new Hadoop (step 3)</DESCRIPTION>
12 
            <STATIC_CONFIGURATION>
13 
				<!-- Cluster wide -->
14 
                <PROPERTY key="queueName" value="default"/>
15 
                <PROPERTY key="user.name" value="dnet"/> <!-- username = sqoop?? -->
16 
				<PROPERTY key="oozie.wf.application.path" value="/user/michele.artini/migration/wfs/regular_step3/oozie_app"/><!-- edit this property! -->
17 
				<PROPERTY key="hbase_dump_distcp_memory_mb" value="6144"/>
18 
				<PROPERTY key="hbase_dump_distcp_num_maps" value="1"/>
19 
            </STATIC_CONFIGURATION>
20 
            <JOB_INTERFACE>
21 
                <PARAM name="migrationPathStep2" required="true" description="migration path - step 2 (source)"/>
22 
                <PARAM name="migrationPathStep3" required="true" description="migration path - step 3 (target)"/>
23 
            </JOB_INTERFACE>
24 
        </HADOOP_JOB>
25 
        <STATUS>
26 
            <LAST_SUBMISSION_DATE value="2014-11-14T19:57:25+00:00"/>
27 
            <RUNNING_INSTANCES value="0"/>
28 
            <CUMULATIVE_RUN value="75"/>
29 
        </STATUS>
30 
        <SECURITY_PARAMETERS>SECURITY_PARAMETERS</SECURITY_PARAMETERS>
31 
    </BODY>
32 
</RESOURCE_PROFILE>

Also available in: Unified diff