Platform For AI:Usar o OSS

Também é possível usar o Python SDK para modificar parâmetros do JindoFuse.

1. Cumpra os seguintes pré-requisitos.

   1. Instale o SDK para AI Workspace.

      !pip install alibabacloud-aiworkspace20210204

   2. Configure variáveis de ambiente. Para mais informações, consulte Install the Credentials Tool e Configure environment variables on Linux, macOS, and Windows.

2. Modifique os parâmetros do JindoFuse.

   Fast read/write

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def change_config():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       # Recommended value: twice the number of CPU cores.
       options['fs.oss.download.thread.concurrency'] = 32
       options['fs.oss.upload.thread.concurrency'] = 32
       options['fs.jindo.args'] = '-oattr_timeout=3 -oentry_timeout=0 -onegative_timeout=0 -oauto_cache -ono_symlink'
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   change_config()

   Incremental read/write

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def change_config():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       # Recommended value: twice the number of CPU cores.
       options['fs.oss.upload.thread.concurrency'] = 32
       options['fs.jindo.args'] = '-oattr_timeout=3 -oentry_timeout=0 -onegative_timeout=0 -oauto_cache -ono_symlink'
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   change_config()

   Consistent read/write

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def change_config():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       options['fs.jindo.args'] = '-oattr_timeout=0 -oentry_timeout=0 -onegative_timeout=0 -oauto_cache -ono_symlink'
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   change_config()

   Read-only

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def change_config():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       # Recommended value: twice the number of CPU cores.
       options['fs.oss.download.thread.concurrency'] = 32
       options['fs.jindo.args'] = '-oro -oattr_timeout=7200 -oentry_timeout=7200 -onegative_timeout=7200 -okernel_cache -ono_symlink'
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   change_config()

   JindoFuse version

   O código abaixo fornece um exemplo:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def change_version():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       # Set the JindoFuse version. Supported versions include 6.4.4, 6.7.0, and 6.6.0. For release notes, see https://aliyun.github.io/alibabacloud-jindodata/releases/.
       options['fs.jindo.fuse.pod.image.tag'] = "6.7.0"
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   change_version()

   Metadata cache

   Ao executar um treinamento distribuído e vários nós tentarem gravar no mesmo diretório simultaneamente, o cache pode causar falhas nas operações de gravação em alguns nós. Para resolver esse problema, adicione o argumento -oattr_timeout=0 -oentry_timeout=0 -onegative_timeout=0 à linha de comando do JindoFuse. O código abaixo fornece um exemplo:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def turnOffMetaCache():
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
       workspace_client = AIWorkspaceClient(
         config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
         )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
   
       options['fs.jindo.args'] = '-oattr_timeout=0 -oentry_timeout=0 -onegative_timeout=0'
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   turnOffMetaCache()
   

   Upload/download thread count

   Defina os seguintes parâmetros para ajustar a contagem de threads:

   • fs.oss.upload.thread.concurrency:32

   • fs.oss.download.thread.concurrency:32

   • fs.oss.read.readahead.buffer.count:64

   • fs.oss.read.readahead.buffer.size:4194304

   O código abaixo fornece um exemplo:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def adjustThreadNum():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
   
       options['fs.oss.upload.thread.concurrency'] = 32
       options['fs.oss.download.thread.concurrency'] = 32
       options['fs.oss.read.readahead.buffer.count'] = 64
    
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
    
    
   adjustThreadNum()
   

   AppendObject writes

   Todos os arquivos criados localmente na montagem são criados como objetos no OSS chamando a API AppendObject. O tamanho de um objeto criado usando AppendObject não pode exceder 5 GB. Para mais limites de uso, consulte AppendObject. O código abaixo fornece um exemplo:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def useAppendObject():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
   
       options['fs.jindo.args'] = '-oattr_timeout=0 -oentry_timeout=0 -onegative_timeout=0'
       options['fs.oss.append.enable'] = "true"
       options['fs.oss.flush.interval.millisecond'] = "1000"
       options['fs.oss.read.buffer.size'] = "262144"
       options['fs.oss.write.buffer.size'] = "262144"
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   useAppendObject()

   OSS-HDFS

   Para informações sobre como habilitar o OSS-HDFS, consulte What is OSS-HDFS service?. O exemplo abaixo mostra como criar um conjunto de dados que usa um endpoint OSS-HDFS:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import CreateDatasetRequest
   
   def createOssHdfsDataset():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       workspace_id = 'YOUR_WORKSPACE_ID'
   
       oss_bucket = 'YOUR_OSS_BUCKET'
       # Use the OSS-HDFS endpoint.
       oss_endpoint = f'{region_id}.oss-dls.aliyuncs.com'
       # The OSS-HDFS path to mount.
       oss_path = '/'
       # The local mount path.
       mount_path = '/mnt/data/'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
   
       response = workspace_client.create_dataset(CreateDatasetRequest(
           workspace_id=workspace_id,
           name="YOUR_DATASET_NAME",
           data_type='COMMON',
           data_source_type='OSS',
           property='DIRECTORY',
           uri=f'oss://{oss_bucket}.{oss_endpoint}{oss_path}',
           accessibility='PRIVATE',
           source_type='USER',
           options=json.dumps({
               'mountPath': mount_path,
               # In distributed training scenarios, we recommend that you add the following parameters.
               'fs.jindo.args': '-oattr_timeout=0 -oentry_timeout=0 -onegative_timeout=0 -ono_symlink -ono_xattr -ono_flock -odirect_io',
               'fs.oss.flush.interval.millisecond': "10000",
               'fs.oss.randomwrite.sync.interval.millisecond': "10000",
           })
       ))
       print(f'datasetId: {response.body.dataset_id}')
   
   createOssHdfsDataset()
   

   Memory resources

   Defina o parâmetro fs.jindo.fuse.pod.mem.limit para ajustar o limite de memória. O código abaixo fornece um exemplo:

   import json
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
   from alibabacloud_aiworkspace20210204.models import UpdateDatasetRequest
   
   def adjustResource():
       # Use the region where the DLC job runs. For example, use cn-hangzhou for China (Hangzhou).
       region_id = 'cn-hangzhou'
       # An Alibaba Cloud account AccessKey has permissions to call all APIs. We recommend that you use a RAM user for API calls and daily O&M.
       # We strongly recommend that you do not store the AccessKey ID or AccessKey Secret in your code. Otherwise, your AccessKey may be leaked and put all resources in your account at risk.
       # In this example, the Credentials SDK reads the AccessKey from environment variables by default for authentication. You must install the Credentials tool and configure environment variables first.
       cred = CredClient()
       dataset_id = 'YOUR_DATASET_ID'
   
       workspace_client = AIWorkspaceClient(
           config=Config(
               credential=cred,
               region_id=region_id,
               endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
           )
       )
       # 1. Get the dataset content.
       get_dataset_resp = workspace_client.get_dataset(dataset_id)
       options = json.loads(get_dataset_resp.body.options)
       # The memory resource to configure.
       options['fs.jindo.fuse.pod.mem.limit'] = "10Gi"
   
       update_request = UpdateDatasetRequest(
           options=json.dumps(options)
       )
       # 2. Update the options.
       workspace_client.update_dataset(dataset_id, update_request)
       print('new options is: {}'.format(update_request.options))
   
   adjustResource()