
    1                     V   S r SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJ	r	  SSK
Jr  SS	KJr  SS
KJr  SSKJr  \	R"                  \	R$                  " \	R&                  R(                  \	R&                  R*                  5       " S S\	R,                  5      5       5       rS rS rS rg)z3Implementation of gcloud dataflow yaml run command.    )apis)storage_api)storage_util)actions)arg_parsers)base)dataflow_util)
properties)yaml)filesc                   8    \ rS rSrSrSSS.r\S 5       rS rSr	g	)
Run   z#Runs a job from the specified path.zERuns a job from the specified YAML description or Cloud Storage path.z          To run a job from YAML, run:

            $ {command} my-job --yaml-pipeline-file=gs://yaml-path --region=europe-west1
          )DESCRIPTIONEXAMPLESc                    U R                  SSSS9  U R                  SS9nUR                  SSS	9  UR                  S
SS	9  U R                  SSS[        R                  -   S9  U R                  SS[        R
                  " 5       [        R                  " SS[        R                  S9SS9  U R                  SSSS9  U R                  SS[        R                  " SS5      S9  U R                  SSS	9  U R                  SS S	9  U R                  S![        R                  " S"S#5      S$S%9  U R                  S&S'[        R                  " 5       [        R                  S(S9  U R                  S)S*[        R                  " 5       [        R                  S+S9  U R                  S,S-[        R
                  " 5       [        R                  S.S9  U R                  S/[        R                  " [        R                  R                  R                  5      S0S19  U R                  S2S3[        R                  " SS5      S9  U R                  S4S5[        R                  " SS5      S9  U R                  S6[         S7S%9  U R                  S8[         S9S%9  U R                  S:S;S	9  U R                  S<S=S	9  U R                  S>S?S	9  U R                  S@[        R                  " [        R                  R                  R"                  5      SAS19  U R%                  5       nUR                  SBSC[        R
                  " 5       [        R                  SDS9  UR                  SESF[        R&                  SSG9  gH)IzfRegister flags for this command.

Args:
  parser: argparse.ArgumentParser to register arguments with.
job_nameJOB_NAMEz!Unique name to assign to the job.)metavarhelpTrequiredz--yaml-pipeline-fileziPath of a file defining the YAML pipeline to run. (Must be a local file or a URL beginning with 'gs://'.))r   z--yaml-pipelinez.Inline definition of the YAML pipeline to run.z--region	REGION_IDz*Region ID of the job's regional endpoint. z--pipeline-optionszOPTIONS=VALUE;OPTION=VALUEzxThe `--pipeline-options` flag is deprecated. Pipeline options should be passed using --additional-pipeline-options flag.)warnactionz$Pipeline options to pass to the job.)r   typer   r   z--jinja-variablesJSON_OBJECTz1Jinja2 variables to be used in reifying the yaml.z--template-file-gcs-locationzbGoogle Cloud Storage location of the YAML template to run. (Must be a URL beginning with 'gs://'.)z^gs://.*zMust begin with 'gs://')r   r   z	--networkztCompute Engine network for launching worker instances to run the pipeline.  If not set, the default network is used.z--subnetworkzzCompute Engine subnetwork for launching worker instances to run the pipeline.  If not set, the default subnetwork is used.z--service-account-emailz	.*@.*\..*z"must provide a valid email addressz&Service account to run the workers as.)r   r   z--additional-experimentsADDITIONAL_EXPERIMENTSzkAdditional experiments to pass to the job. Example: --additional-experiments=experiment1,experiment2=value2z--additional-pipeline-optionsADDITIONAL_PIPELINE_OPTIONSzAdditional pipeline options to pass to the job. Example: --additional-pipeline-options=option1=value1,option2=value2 For a list of available options, see the Dataflow reference: https://cloud.google.com/dataflow/docs/reference/pipeline-optionsz--additional-user-labelsADDITIONAL_USER_LABELSzfAdditional user labels to pass to the job. Example: --additional-user-labels='key1=value1,key2=value2'z--disable-public-ipszFIf specified, Cloud Dataflow workers will not use public IP addresses.)r   r   z--staging-locationzGoogle Cloud Storage location to stage local files. If not set, defaults to the value for --temp-location.(Must be a URL beginning with 'gs://'.)z--temp-locationzGoogle Cloud Storage location to stage temporary files. If not set, defaults to the value for --staging-location.(Must be a URL beginning with 'gs://'.)z--max-workersz!Maximum number of workers to run.z--num-workersz!Initial number of workers to use.z--worker-machine-typezAType of machine to use for workers. Defaults to server-specified.z--launcher-machine-typezLThe machine type to use for launching the job. The default is n1-standard-1.z--dataflow-kms-keyz+Cloud KMS key to protect the job resources.z--enable-streaming-enginez.Enable Streaming Engine for the streaming job.z--transform-name-mappingsTRANSFORM_NAME_MAPPINGSz5Transform name mappings for the streaming update job.z--updatez,Specify this flag to update a streaming job.)r   r   r   N)add_argumentadd_mutually_exclusive_groupr	   DEFAULT_REGION_MESSAGEr   ArgDictr   DeprecationActionUpdateActionRegexpValidatorArgListStoreBooleanPropertyr
   VALUESdataflowdisable_public_ipsintenable_streaming_engineadd_argument_groupStoreTrueFalseAction)parsergroupstreaming_update_argss       lib/surface/dataflow/yaml/run.pyArgsRun.Args.   s     J-P   ///>E	F   
 P   8223	   ,  "(( M ++- 4   @   &8(()DF	   F   M   !(()MO 5   "(  "''F  	 '-  "''P   "(  "''A  	 ++&&99
  	 6 ((2
  
 6 ((2
  
 c(K   c(K   O   !   #P   #++&&>>
 >   #557&&#)  "''	@ ' C &&<//	 '     c           	      B   [        UR                  =(       d    0 5      nUR                  (       a-  [        UR                  5      nUc  UR                  US'   OX2S'   OUR                  US'   UR
                  (       a  UR
                  US'   SU;   a  SU;  a  [        US   5        [        UR                  S5      U5      nUR                  =(       d)    [        R                  R                  R                  U5      n[        R                  " S0 S[        R                   R"                  R$                  R'                  SS9_SU_S	UR(                  _S
U_SU_SUR*                  _SUR,                  _SUR.                  _SUR0                  _SUR2                  _SUR4                  _S[        R                   R6                  R8                  R;                  5       _SUR<                  _SUR>                  _SUR@                  _SURB                  _SURD                  _SURF                  _SURH                  _S[        R                   R6                  RJ                  R;                  5       _SURL                  _SURN                  _6n[        R                  RQ                  U5      $ )zyRuns the command.

Args:
  args: The arguments that were provided to this command invocation.

Returns:
  A Job message.
yaml_pipeline_fileyaml_pipelinejinja_variableszjinja-variables
project_idTr   	region_idr   gcs_location
parametersnetwork
subnetworkservice_account_emailadditional_experimentsadditional_pipeline_optionsadditional_user_labelsr-   staging_locationtemp_locationmax_workersnum_workersworker_machine_typelauncher_machine_typekms_key_namer/   transform_name_mappingsstreaming_update ))dictpipeline_optionsr:   _try_get_yaml_contentsr;   r<   _validate_yaml _get_region_from_yaml_or_defaultgettemplate_file_gcs_locationr   	TemplatesYAML_TEMPLATE_GCS_LOCATIONformatTemplateArgumentsr
   r+   coreprojectGetr   rA   rB   rC   rD   rE   rF   r,   r-   GetBoolrG   rH   rI   rJ   rK   rL   dataflow_kms_keyr/   rN   updateCreateJobFromFlexTemplate)selfargsr@   yaml_contentsr>   r?   	argumentss          r5   r   Run.Run   s    d++1r2J ,T-D-DEm		+/+B+B
'(&3?# %)$6$6j!&*&:&:j"#*$):*)LZ010'I
 	'' 	G>>44;;IF 
 && $$))1155t5D  "	
   ?? #88  $:: %)$D$D  $:: &,,55HHPPR .. (( $$  $$!" !44#$ #88%& **'( !+ 1 1 : : R R Z Z \)* !% < <+, -I0 >>33I>>r8   rP   N)
__name__
__module____qualname____firstlineno____doc__detailed_helpstaticmethodr6   r   __static_attributes__rP   r8   r5   r   r      s4     , 
- K KZ=?r8   r   c                 n     [         R                  " U 5      ng ! [         a  n[        S5      UeS nAff = f)Nz#yaml_pipeline must be a valid yaml.)r   load	Exception
ValueError)r;   _exns      r5   rT   rT   <  s6    E		- A	 E
:
;DEs    
4/4c                 $   UR                   nSn [        R                  " U 5      nU(       d  [        R                  " U5      $ SU;   aB  SUS   ;   a9  US   S   nSU;   d  SU;   a%  [        R
                  " SR                  U5      5      eU(       a(  U(       a  X#:w  a  [        SR                  X25      5      eU$ [        R                  " U5      $ ! [        R
                   a+  nU(       d  [        SR                  U5      5         SnANSnAff = f)	zEGets the region from yaml pipeline or args, or falls back to default.Noptionsregion{}zlyaml pipeline contains unparsable region: {0}. Found curly braces in region. Falling back to default region.zyFailed to get region from yaml pipeline: {0}. If using jinja variables, parsing may fail. Falling back to default region.zeRegion specified in yaml pipeline options ({0}) does not match region specified in command line ({1}))	rx   r   rq   r	   	GetRegionYAMLParseErrorrZ   printrs   )r;   rd   rx   options_regionpipeline_dataru   s         r5   rU   rU   E  s   ;;&.IIm,M$$T**M!h-	2J&J$Y/9n		#"7!!99?9O
 	
 &*44:F5  		 	 	&&% 
		 F3Ks   2C AC D$!D

Dc                    U R                  S5      (       d  [        R                  " U 5      $ [        R                  " 5       n[
        R                  R                  U 5      n UR                  U5      R                  5       R                  S5      $ ! [         a$  n[        SR                  X5      5         SnAgSnAff = f)z@Reads yaml contents from the specified file if it is accessable.zgs://zutf-8zUnable to read file {0} due to incorrect file path or insufficient read permissions. Will not be able to validate the yaml pipeline or determine the region from the yaml pipeline options. Error: {1}N)
startswithr   ReadFileContentsr   StorageClientr   ObjectReferenceFromUrl
ReadObjectreaddecoderr   r}   rZ   )r:   storage_clientobj_refes       r5   rS   rS   i  s    		&	&w	/	/!!"455,,..((001CD'$$W-224;;GDD	 		  &v&8<	  
s   "-B 
B>B99B>N)rl   googlecloudsdk.api_lib.dataflowr   googlecloudsdk.api_lib.storager   r   googlecloudsdk.callioper   r   r   #googlecloudsdk.command_lib.dataflowr	   googlecloudsdk.corer
   r   googlecloudsdk.core.utilr   DefaultUniverseOnlyReleaseTracksReleaseTrackGABETACommandr   rT   rU   rS   rP   r8   r5   <module>r      s    : 0 6 7 + / ( = * $ * D%%(($*;*;*@*@AZ?$,, Z? B Z?zE!'Hr8   