Platform For AI:Uso de OSS

También puede usar el SDK de Python para modificar los parámetros de JindoFuse.

1. Complete los siguientes requisitos previos.

   1. Instale el SDK para AI Workspace.

      !pip install alibabacloud-aiworkspace20210204

   2. Configure las variables de entorno. Para más información, consulte Install the Credentials Tool y Configure environment variables on Linux, macOS, and Windows.

2. Modifique los parámetros de JindoFuse.

   Fast read/write

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def change_config():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       # Recommended value: twice the number of CPU cores.
       options['fs.oss.download.thread.concurrency'] = 32
       options['fs.oss.upload.thread.concurrency'] = 32
       options['fs.jindo.args'] = '-oattr_timeout=3 -oentry_timeout=0 -onegative_timeout=0 -oauto_cache -ono_symlink'
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   change_config()

   Incremental read/write

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def change_config():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       # Recommended value: twice the number of CPU cores.
       options['fs.oss.upload.thread.concurrency'] = 32
       options['fs.jindo.args'] = '-oattr_timeout=3 -oentry_timeout=0 -onegative_timeout=0 -oauto_cache -ono_symlink'
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   change_config()

   Consistent read/write

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def change_config():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       options['fs.jindo.args'] = '-oattr_timeout=0 -oentry_timeout=0 -onegative_timeout=0 -oauto_cache -ono_symlink'
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   change_config()

   Read-only

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def change_config():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       # Recommended value: twice the number of CPU cores.
       options['fs.oss.download.thread.concurrency'] = 32
       options['fs.jindo.args'] = '-oro -oattr_timeout=7200 -oentry_timeout=7200 -onegative_timeout=7200 -okernel_cache -ono_symlink'
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   change_config()

   JindoFuse version

   El siguiente código proporciona un ejemplo:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def change_version():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       # Set the JindoFuse version. Supported versions include 6.4.4, 6.7.0, and 6.6.0. For release notes, see https://aliyun.github.io/alibabacloud-jindodata/releases/.
       options['fs.jindo.fuse.pod.image.tag'] = "6.7.0"
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   change_version()

   Metadata cache

   Al ejecutar un entrenamiento distribuido donde varios nodos intentan escribir en el mismo directorio simultáneamente, la caché podría causar fallos en las operaciones de escritura de algunos nodos. Para resolver este problema, agregue el argumento -oattr_timeout=0 -oentry_timeout=0 -onegative_timeout=0 a la línea de comandos de JindoFuse. El siguiente código proporciona un ejemplo:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def turnOffMetaCache():
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
       workspace_client = AIWorkspaceClient(
         config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
         )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
   
       options['fs.jindo.args'] = '-oattr_timeout=0 -oentry_timeout=0 -onegative_timeout=0'
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   turnOffMetaCache()
   

   Upload/download thread count

   Establezca los siguientes parámetros para ajustar el recuento de hilos:

   • fs.oss.upload.thread.concurrency:32

   • fs.oss.download.thread.concurrency:32

   • fs.oss.read.readahead.buffer.count:64

   • fs.oss.read.readahead.buffer.size:4194304

   El siguiente código proporciona un ejemplo:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def adjustThreadNum():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
   
       options['fs.oss.upload.thread.concurrency'] = 32
       options['fs.oss.download.thread.concurrency'] = 32
       options['fs.oss.read.readahead.buffer.count'] = 64
    
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
    
    
   adjustThreadNum()
   

   AppendObject writes

   Todos los archivos creados localmente en el punto de montaje se crean como objetos en OSS mediante la llamada a la API AppendObject. El tamaño de un objeto creado con AppendObject no puede superar los 5 GB. Para más límites de uso, consulte AppendObject. El siguiente código proporciona un ejemplo:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def useAppendObject():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
   
       options['fs.jindo.args'] = '-oattr_timeout=0 -oentry_timeout=0 -onegative_timeout=0'
       options['fs.oss.append.enable'] = "true"
       options['fs.oss.flush.interval.millisecond'] = "1000"
       options['fs.oss.read.buffer.size'] = "262144"
       options['fs.oss.write.buffer.size'] = "262144"
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   useAppendObject()

   OSS-HDFS

   Para obtener información sobre cómo habilitar OSS-HDFS, consulte What is OSS-HDFS service?. El siguiente ejemplo muestra cómo crear un conjunto de datos que utiliza un endpoint de OSS-HDFS:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import CreateDatasetRequest
   
   def createOssHdfsDataset():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       workspace_id = 'YOUR_WORKSPACE_ID'
   
       oss_bucket = 'YOUR_OSS_BUCKET'
       # Use the OSS-HDFS endpoint.
       oss_endpoint = f'{region_id}.oss-dls.aliyuncs.com'
       # The OSS-HDFS path to mount.
       oss_path = '/'
       # The local mount path.
       mount_path = '/mnt/data/'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
   
       response = workspace_client.create_dataset(CreateDatasetRequest(
           workspace_id=workspace_id,
           name="YOUR_DATASET_NAME",
           data_type='COMMON',
           data_source_type='OSS',
           property='DIRECTORY',
           uri=f'oss://{oss_bucket}.{oss_endpoint}{oss_path}',
           accessibility='PRIVATE',
           source_type='USER',
           options=json.dumps({
               'mountPath': mount_path,
               # In distributed training scenarios, we recommend that you add the following parameters.
               'fs.jindo.args': '-oattr_timeout=0 -oentry_timeout=0 -onegative_timeout=0 -ono_symlink -ono_xattr -ono_flock -odirect_io',
               'fs.oss.flush.interval.millisecond': "10000",
               'fs.oss.randomwrite.sync.interval.millisecond': "10000",
           })
       ))
       print(f'datasetId: {response.body.dataset_id}')
   
   createOssHdfsDataset()
   

   Memory resources

   Puede establecer el parámetro fs.jindo.fuse.pod.mem.limit para ajustar el límite de memoria. El siguiente código proporciona un ejemplo:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def adjustResource():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       # The memory resource to configure.
       options['fs.jindo.fuse.pod.mem.limit'] = "10Gi"
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   adjustResource()