E-MapReduce:Create and manage a project

Scenarios

If you want to create and edit jobs for an EMR cluster, you must first create a project in Data Platform. Then, you can create, edit, and schedule jobs in the project. After you create a project, you can associate cluster resources with the project and add members to it.

Examples

• Java
  1. Create a project named emr_openapi_demo_project in the China (Hangzhou) region.

     import com.aliyuncs.DefaultAcsClient;
     import com.aliyuncs.IAcsClient;
     import com.aliyuncs.exceptions.ClientException;
     import com.aliyuncs.exceptions.ServerException;
     import com.aliyuncs.profile.DefaultProfile;
     import com.google.gson.Gson;
     import java.util.*;
     import com.aliyuncs.emr.model.v20160408.*;
     
     public class CreateFlowProject {
     
         public static void main(String[] args) {
             DefaultProfile profile = DefaultProfile.getProfile("cn-hangzhou", "<accessKeyId>", "<accessSecret>");
             IAcsClient client = new DefaultAcsClient(profile);
     
             CreateFlowProjectRequest request = new CreateFlowProjectRequest();
             request.setRegionId("cn-hangzhou");
             request.setName("emr_openapi_demo_project");
             request.setDescription("Create flow project via CreateFlowProject");
     
             try {
                 CreateFlowProjectResponse response = client.getAcsResponse(request);
                 System.out.println(new Gson().toJson(response));
             } catch (ServerException e) {
                 e.printStackTrace();
             } catch (ClientException e) {
                 System.out.println("ErrCode:" + e.getErrCode());
                 System.out.println("ErrMsg:" + e.getErrMsg());
                 System.out.println("RequestId:" + e.getRequestId());
             }
     
         }
     }

     Find the ID of the project in the returned information.

     {    
         "RequestId": "BF47231B-29AE-4A4F-A1A3-10205ECD9634",
         "Id": "FP-D18E9976D5A5****"
     }

  2. Query the clusters that can be associated with the project in the current region.

     import com.aliyuncs.DefaultAcsClient;
     import com.aliyuncs.IAcsClient;
     import com.aliyuncs.exceptions.ClientException;
     import com.aliyuncs.exceptions.ServerException;
     import com.aliyuncs.profile.DefaultProfile;
     import com.google.gson.Gson;
     import java.util.*;
     import com.aliyuncs.emr.model.v20160408.*;
     
     public class ListFlowClusterAll {
     
         public static void main(String[] args) {
             DefaultProfile profile = DefaultProfile.getProfile("cn-hangzhou", "<accessKeyId>", "<accessSecret>");
             IAcsClient client = new DefaultAcsClient(profile);
     
             ListFlowClusterAllRequest request = new ListFlowClusterAllRequest();
             request.setRegionId("cn-hangzhou");
     
             try {
                 ListFlowClusterAllResponse response = client.getAcsResponse(request);
                 System.out.println(new Gson().toJson(response));
             } catch (ServerException e) {
                 e.printStackTrace();
             } catch (ClientException e) {
                 System.out.println("ErrCode:" + e.getErrCode());
                 System.out.println("ErrMsg:" + e.getErrMsg());
                 System.out.println("RequestId:" + e.getRequestId());
             }
     
         }
     }

     Find the ID of the target cluster in the returned information. For example, if the cluster information is returned in the JSON format, find the cluster ID based on the following path:

     Clusters -> ClusterInfo -> Id -> Cluster ID

  3. Associate the target cluster with the emr_openapi_demo_project project. This way, workflows in the project run in the cluster. You can also set the queue to which jobs are submitted, the users who can submit jobs, and the instances on which jobs run. Jobs can run only on gateway and master instances.

     import com.aliyuncs.DefaultAcsClient;
     import com.aliyuncs.IAcsClient;
     import com.aliyuncs.exceptions.ClientException;
     import com.aliyuncs.exceptions.ServerException;
     import com.aliyuncs.profile.DefaultProfile;
     import com.google.gson.Gson;
     import java.util.*;
     import com.aliyuncs.emr.model.v20160408.*;
     
     public class CreateFlowProjectClusterSetting {
     
         public static void main(String[] args) {
             DefaultProfile profile = DefaultProfile.getProfile("cn-hangzhou", "<accessKeyId>", "<accessSecret>");
             IAcsClient client = new DefaultAcsClient(profile);
     
             CreateFlowProjectClusterSettingRequest request = new CreateFlowProjectClusterSettingRequest();
             request.setRegionId("cn-hangzhou");
             request.setProjectId("FP-D18E9976D5A5****");
             request.setClusterId("C-B503DDB15B34****");
     
             try {
                 CreateFlowProjectClusterSettingResponse response = client.getAcsResponse(request);
                 System.out.println(new Gson().toJson(response));
             } catch (ServerException e) {
                 e.printStackTrace();
             } catch (ClientException e) {
                 System.out.println("ErrCode:" + e.getErrCode());
                 System.out.println("ErrMsg:" + e.getErrMsg());
                 System.out.println("RequestId:" + e.getRequestId());
             }
     
         }
     }

  4. Add a RAM user to the project.
     The following sample code shows how to add user test to the emr_openapi_demo_project project and grant user test the permissions to develop the project:

     import com.aliyuncs.DefaultAcsClient;
     import com.aliyuncs.IAcsClient;
     import com.aliyuncs.exceptions.ClientException;
     import com.aliyuncs.exceptions.ServerException;
     import com.aliyuncs.profile.DefaultProfile;
     import com.google.gson.Gson;
     import java.util.*;
     import com.aliyuncs.emr.model.v20160408.*;
     
     public class CreateFlowProjectUser {
     
         public static void main(String[] args) {
             DefaultProfile profile = DefaultProfile.getProfile("cn-hangzhou", "<accessKeyId>", "<accessSecret>");
             IAcsClient client = new DefaultAcsClient(profile);
     
             CreateFlowProjectUserRequest request = new CreateFlowProjectUserRequest();
             request.setRegionId("cn-hangzhou");
             request.setProjectId("FP-D18E9976D5A5****");
     
             List<CreateFlowProjectUserRequest.User> userList = new ArrayList<CreateFlowProjectUserRequest.User>();
     
             CreateFlowProjectUserRequest.User user1 = new CreateFlowProjectUserRequest.User();
             user1.setUserId(" 24339807068856****");
             user1.setUserName("test");
             userList.add(user1);
             request.setUsers(userList);
     
             try {
                 CreateFlowProjectUserResponse response = client.getAcsResponse(request);
                 System.out.println(new Gson().toJson(response));
             } catch (ServerException e) {
                 e.printStackTrace();
             } catch (ClientException e) {
                 System.out.println("ErrCode:" + e.getErrCode());
                 System.out.println("ErrMsg:" + e.getErrMsg());
                 System.out.println("RequestId:" + e.getRequestId());
             }
     
         }
     }

• Python
  1. Create a project named emr_openapi_demo_project in the China (Hangzhou) region.

     #! /usr/bin/env python
     #coding=utf-8
     
     from aliyunsdkcore.client import AcsClient
     from aliyunsdkcore.acs_exception.exceptions import ClientException
     from aliyunsdkcore.acs_exception.exceptions import ServerException
     from aliyunsdkemr.request.v20160408.CreateFlowProjectRequest import CreateFlowProjectRequest
     
     client = AcsClient('<accessKeyId>', '<accessSecret>', 'cn-hangzhou')
     
     request = CreateFlowProjectRequest()
     request.set_accept_format('json')
     
     request.set_Name("emr_openapi_demo_project")
     request.set_Description("Create flow project via CreateFlowProject")
     
     response = client.do_action_with_exception(request)
     # python2:  print(response) 
     print(str(response, encoding='utf-8'))

     Find the ID of the project in the returned information.

     {    
         "RequestId": "BF47231B-29AE-4A4F-A1A3-10205ECD9634",
         "Id": "FP-D18E9976D5A5****"
     }

  2. Query the clusters that can be associated with the project in the current region.

     #! /usr/bin/env python
     #coding=utf-8
     
     from aliyunsdkcore.client import AcsClient
     from aliyunsdkcore.acs_exception.exceptions import ClientException
     from aliyunsdkcore.acs_exception.exceptions import ServerException
     from aliyunsdkemr.request.v20160408.ListFlowClusterRequest import ListFlowClusterRequest
     
     client = AcsClient('<accessKeyId>', '<accessSecret>', 'cn-hangzhou')
     
     request = ListFlowClusterRequest()
     request.set_accept_format('json')
     
     request.set_ProjectId("FP-D18E9976D5A5****")
     
     response = client.do_action_with_exception(request)
     # python2:  print(response) 
     print(str(response, encoding='utf-8'))

     Find the ID of the target cluster in the returned information. For example, if the cluster information is returned in the JSON format, find the cluster ID based on the following path:

     Clusters -> ClusterInfo -> Id -> Cluster ID

  3. Associate the target cluster with the emr_openapi_demo_project project. This way, workflows in the project run in the cluster. You can also set the queue to which jobs are submitted, the users who can submit jobs, and the instances on which jobs run. Jobs can run only on gateway and master instances.

     #! /usr/bin/env python
     #coding=utf-8
     
     from aliyunsdkcore.client import AcsClient
     from aliyunsdkcore.acs_exception.exceptions import ClientException
     from aliyunsdkcore.acs_exception.exceptions import ServerException
     from aliyunsdkemr.request.v20160408.CreateFlowProjectClusterSettingRequest import CreateFlowProjectClusterSettingRequest
     
     client = AcsClient('<accessKeyId>', '<accessSecret>', 'cn-hangzhou')
     
     request = CreateFlowProjectClusterSettingRequest()
     request.set_accept_format('json')
     
     request.set_ProjectId("FP-D18E9976D5A5****")
     request.set_ClusterId("C-B503DDB15B34****")
     
     response = client.do_action_with_exception(request)
     # python2:  print(response) 
     print(str(response, encoding='utf-8'))

  4. Add a RAM user to the project.
     The following sample code shows how to add user test to the emr_openapi_demo_project project and grant user test the permissions to develop the project.

     #! /usr/bin/env python
     #coding=utf-8
     
     from aliyunsdkcore.client import AcsClient
     from aliyunsdkcore.acs_exception.exceptions import ClientException
     from aliyunsdkcore.acs_exception.exceptions import ServerException
     from aliyunsdkemr.request.v20160408.CreateFlowProjectUserRequest import CreateFlowProjectUserRequest
     
     client = AcsClient('<accessKeyId>', '<accessSecret>', 'cn-hangzhou')
     
     request = CreateFlowProjectUserRequest()
     request.set_accept_format('json')
     
     request.set_ProjectId("FP-D18E9976D5A5****")
     request.set_Users([
       {
         "UserId": " 24339807068856****",
         "UserName": "test"
       }
     ])
     
     response = client.do_action_with_exception(request)
     # python2:  print(response) 
     print(str(response, encoding='utf-8'))

Related API operations

• Create a data development project
• Query the list of clusters available for data development
• Create project cluster settings
• Add a project user