o
    !de                     @   s   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm	Z	 d d	lm
Z
 d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ G dd deZdS )    N)BasicCommand)applicationutils)argumentschema)	constants)
emrfsutils)emrutils)
exceptions)
hbaseutils)helptext)instancegroupsutils)instancefleetsutils)	steputils)Command)EC2_ROLE_NAME)EMR_ROLE_NAME)jsonc                   @   s  e Zd ZdZejZg dejddejddej	dde
jejddejdd	ejdd
ddejdddddde
jejdddejddejddejddejddejdddejddejddeje
jddddejdddddd ejdd!dd"ejdd#dd"dd$dd%ejdd&dd%dd'd(eje
j d)d*ej!e
j"dd+ej#e
j$dd,ej%e
j&dd-e
j'ej(dd.ej)dd/e
j*ej+dd0ej,dd1ej-dd2ej.dd3ej/dd4e
j0ej1dd5d6ej2d7d8e
j3ej4dd9e
j5ej6dd:e
j7ej8dZ9e:;d;d<Z<e:;d;d=Z=d>d? Z>d@dA Z?dBdC Z@dDdE ZAdFdG ZBdHdI ZCdJdK ZDdLdM ZEdNdO ZFdPdQ ZGdRdS ZHdTdU ZIdVS )WCreateClusterzcreate-clusterzrelease-label)name	help_textzos-release-labelzami-versionzinstance-groups)r   schemar   zinstance-typezinstance-countzauto-terminate
store_trueauto_terminate)r   action
group_namer   zno-auto-terminate)r   r   r   zinstance-fleetsr   zDevelopment Cluster)r   defaultr   zlog-urizlog-encryption-kms-key-idzservice-rolezauto-scaling-rolezuse-default-roles)r   r   r   configurationszec2-attributes)r   r   r   ztermination-protectedtermination_protectedzno-termination-protectedzscale-down-behaviorzvisible-to-all-usersZ
visibilityzno-visible-to-all-userszenable-debuggingdebugzno-enable-debuggingtags+)r   nargsr   r   zbootstrap-actionsapplicationsemrfsstepszadditional-infozrestore-from-hbase-backupzsecurity-configurationzcustom-ami-idzebs-root-volume-sizezrepo-upgrade-on-bootzkerberos-attributeszstep-concurrency-levelinteger)r   Zcli_type_namer   zmanaged-scaling-policyzplacement-group-configszauto-termination-policyZemrzcreate-cluster-synopsis.txtzcreate-cluster-examples.rstc                 C   s  i }|j |d< | | d}|jdu r |jd ur tjdd|d|jdu r7|jd ur7d|jv r7tjdd|d|jd urH|jd urHtjd	d
di }|jd urXt	
|j|d< ntj|j|j|jd|d< |jd ur|j|d< |jd urz
t|j|d< W n ty   tdw |jd u r|jd urtd|j}|d u rtj|jd|j|d< t|d|j t|d|j |jd urt|d|j |jd urt|d|j |jdu rt|_|jd u ri |_t|jd< t|d|j |jd ur|d D ]}d|  v r|j!d u rt" qt|d|j! |j#d ur&t|d|j# |j$du r5|j%du r5d|_$t&|j$d|j%d|d < t&|j'd!|j(d"|d#< |j)du r\|j*du r\d|_)t&|j)d$|j*d%|d&< t+|j,|d'< ||d(< |jd ur| j-||jd) t&|j.d*|j/d+}|jd u r|du rtj0|du r| j1|d,| 2||gd- |j3d ur|jd u rt4j5| j6|j3|d d.\}	}
}| 1|d/|	 | 1|d0|
 | 1|d,| ng |d1< |j3D ]
}|d1 7| q|j8}|d urt9j:|;d2|;d3d4}tj<t=j>t=j?t=j@|d5}| 1|d,|g |jAd ur&| jB||jAd6 |jCd ur6| jD||jC|jd7 |jEd urOtFjG|jE| j6|jd8}| j1|d,|d- |jHd ur]t|d9|jH |jId urkt|d:|jI |jJd ur{t|d;tK|jJ |jLd urt|d<|jL |jMd urt|d=|jM |jNd ur|jN|d>< |jOd urt|d?|jO |jPd urt|d@|jP |jQd urt|dA|jQ | R| tS| jTdB|| j6|jU|jV}| W|}tX| jTdB|| dCS )DNNamez| Either choose --use-default-roles or use both --service-role <roleName> and --ec2-attributes InstanceProfile=<profileName>.Tz--use-default-rolesz--service-role)option1option2messageInstanceProfilez --ec2-attributes InstanceProfilez--instance-groupsz--instance-fleetsr&   r'   ZInstanceFleets)instance_groupsinstance_typeinstance_countZInstanceGroupsZReleaseLabelConfigurationsz=aws: error: invalid json argument for option --configurationsz\d?\..*)ami_versionZ
AmiVersionZAdditionalInfoZLogUriZOSReleaseLabelZLogEncryptionKmsKeyIdZServiceRoleZAutoScalingPolicyZAutoScalingRoleZScaleDownBehaviorFz--no-auto-terminatez--auto-terminateZKeepJobFlowAliveWhenNoStepsz--termination-protectedz--no-termination-protectedZTerminationProtectedz--visible-to-all-usersz--no-visible-to-all-usersZVisibleToAllUsersZTags	Instances)clusterparsed_attrsz--enable-debuggingz--no-enable-debuggingZStepsr1   keyvalue)regionZparsed_applicationsr/   ZNewSupportedProductsBootstrapActionsZApplicationsZDirZBackupVersion)dirZbackup_version)jarr   action_on_failureargs)r1   parsed_boostrap_actions)r1   
emrfs_argsrelease_label)Zparsed_step_listr6   r>   ZSecurityConfigurationZCustomAmiIdZEbsRootVolumeSizeZRepoUpgradeOnBootZKerberosAttributesZStepConcurrencyLevelZManagedScalingPolicyZPlacementGroupConfigsZAutoTerminationPolicyZrun_job_flowr   )Yr   #_validate_release_label_ami_versionuse_default_rolesservice_roler   MutualExclusiveOptionErrorZec2_attributesr+   Zinstance_fleetsr   Z"validate_and_build_instance_fleetsr   Z"validate_and_build_instance_groupsr,   r-   r>   r   r   loads
ValueErrorr/   rematchZInvalidAmiVersionErrorr   Z
apply_dictZadditional_infoZlog_uriZos_release_labelZlog_encryption_kms_key_idr   r   keysZauto_scaling_roleZMissingAutoScalingRoleErrorZscale_down_behaviorZno_auto_terminater   Zapply_boolean_optionsr   Zno_termination_protectedZvisible_to_all_usersZno_visible_to_all_usersZ
parse_tagsr   _build_ec2_attributesZenable_debuggingZno_enable_debuggingZLogUriError_update_cluster_dict_build_enable_debuggingr!   r   Zbuild_applicationsr6   appendrestore_from_hbase_backupr	   Z$build_hbase_restore_from_backup_argsget
build_stepr   ZHBASE_JAR_PATHZHBASE_RESTORE_STEP_NAMEZCANCEL_AND_WAITbootstrap_actions_build_bootstrap_actionsr"   _handle_emrfs_parametersr#   r   Zbuild_step_config_listZsecurity_configurationZcustom_ami_idZebs_root_volume_sizeintZrepo_upgrade_on_bootZkerberos_attributesZstep_concurrency_levelZmanaged_scaling_policyZplacement_group_configsZauto_termination_policy_validate_required_applicationscallZ_sessionZendpoint_urlZ
verify_ssl_construct_resultZdisplay_response)selfparsed_argsparsed_globalsparamsZservice_role_validation_messageZinstances_configZis_valid_ami_versionZinstance_groupZdebugging_enabledZapp_listZba_listZ	step_listZapplicationZhbase_restore_configr;   Zstep_configZ
steps_listZrun_job_flow_responseZconstructed_result rZ   Kusr/lib/python3.10/site-packages/awscli/customizations/emr/createcluster.py_run_main_command   s  




























zCreateCluster._run_main_commandc                 C   s:   d }d }|d ur| d}| d}|d ur||dS i S )NZ	JobFlowId
ClusterArn)Z	ClusterIdr]   )rM   )rV   Zrun_job_flow_resultZ	jobFlowIdZ
clusterArnrZ   rZ   r[   rU     s   

zCreateCluster._construct_resultc                 C   sx  |  }|d }d|v rd|v rtjdddd|v r&d|v r&tjdddd|v s.d|v r9d|v s6d|v r9tjtj|d|dd	 tj|d|d
d	 tj|d|dd	 d|v rht |d< tj|d|d dd	 d|v r|t |d< tj|d|d dd	 tj|d|dd	 tj|d|dd	 tj|d|dd	 tj|d|dd	 tj|d|dd	 tj|d|dd	 tj|d|d |S )Nr0   ZSubnetIdZ	SubnetIdsr*   ZAvailabilityZoneZAvailabilityZonesZKeyNameZ
Ec2KeyNameZ
src_paramsZsrc_keyZdest_paramsZdest_keyZEc2SubnetIdZEc2SubnetIdsZ	Placementr)   ZJobFlowRoleZEmrManagedMasterSecurityGroupZEmrManagedSlaveSecurityGroupZServiceAccessSecurityGroupZAdditionalMasterSecurityGroupsZAdditionalSlaveSecurityGroupsrY   r4   r5   )rG   r   rB   ZSubnetAndAzValidationErrorr   apply_paramsdictapply)rV   r1   r2   rG   Z	instancesrZ   rZ   r[   rH     s   

z#CreateCluster._build_ec2_attributesc           	      C   s   | d}|d u rg }g }t|t| tjkrtd|D ]9}i }| dd ur1| d|d< ntj|d< i }tj|d|dd tj|d|dd tj|d|d |	| q|| }|rb||d< |S )	Nr7   zGaws: error: maximum number of bootstrap actions for a cluster exceeded.r%   Pathr^   ZArgsZScriptBootstrapActionr_   )
rM   lenr   ZMAX_BOOTSTRAP_ACTION_NUMBERrD   ZBOOTSTRAP_ACTION_NAMEr   r`   rb   rK   )	rV   r1   r<   Zcluster_ba_listrO   baZ	ba_configZscript_arg_configresultrZ   rZ   r[   rP     s@   

z&CreateCluster._build_bootstrap_actionsc                 C   sL   |j rtj}tjg}nt| j}tjtj| jdg}tj	tj
tj||dS )N)Zrelative_pathr6   )r   r:   r9   r;   )r>   r   ZCOMMAND_RUNNERZDEBUGGING_COMMANDr   Zget_script_runnerr6   Zbuild_s3_linkZDEBUGGING_PATHrN   ZDEBUGGING_NAMEZTERMINATE_CLUSTER)rV   rW   rX   r9   r;   rZ   rZ   r[   rJ     s   
z%CreateCluster._build_enable_debuggingc                 C   s,   ||v r||  |7  < |S |r|||< |S )NrZ   )rV   r1   r4   r5   rZ   rZ   r[   rI   	  s   z"CreateCluster._update_cluster_dictc                 C   sL   |j d ur|jd urtjddd|j d u r"|jd u r$tjdddd S d S )Nz--ami-versionz--release-labelr*   )r/   r>   r   rB   ZRequiredOptionsError)rV   rW   rZ   rZ   r[   r?     s   



z1CreateCluster._validate_release_label_ami_versionc                 C   sj   t g }|jd urt dd |jD }| ||}|jd ur+tj|vr+|tj  |r3tj	|dd S )Nc                 S   s   g | ]}|d    qS )r%   )lower).0ZapprZ   rZ   r[   
<listcomp>$  s    zACreateCluster._validate_required_applications.<locals>.<listcomp>)r!   )
setr!   #_get_missing_applications_for_stepsrL   r   ZHBASEaddtitler   ZMissingApplicationsError)rV   rW   specified_appsmissing_appsrZ   rZ   r[   rS     s   


z-CreateCluster._validate_required_applicationsc                 C   s   t tjtjtjg}t  }|jd urA|jD ]+}t|t|kr" |S |d}|d ur@| }||v r@||vr@|	|d 
  q|S )NType)rj   r   ZHIVEZPIGZIMPALAr#   rd   rM   rg   rl   rm   )rV   rn   rW   Zallowed_app_stepsro   stepZ	step_typerZ   rZ   r[   rk   1  s"   



z1CreateCluster._get_missing_applications_for_stepsc                 C   s   |j r
dd |D }|S )Nc                 S   s$   g | ]}|j d kr|j dkr|qS )rA   Zinstance_profile)r   )rh   xrZ   rZ   r[   ri   E  s
    

zICreateCluster._filter_configurations_in_special_cases.<locals>.<listcomp>)r@   )rV   r   rW   Zparsed_configsrZ   rZ   r[   '_filter_configurations_in_special_casesB  s   z5CreateCluster._filter_configurations_in_special_casesc                 C   sP   |r|  | t|}| j|d|gd d S t| j|}| j|d|d d S )Nr.   r3   r7   )validate_no_emrfs_configurationr   Zbuild_emrfs_confiurationrI   Zbuild_bootstrap_action_configsr6   )rV   r1   r=   r>   Zemrfs_configurationZemrfs_ba_config_listrZ   rZ   r[   rQ   J  s    


z&CreateCluster._handle_emrfs_parametersc                 C   s<   d|v r|d D ]}|d ur| dtjkrtjqd S d S )Nr.   ZClassification)rM   r   Z
EMRFS_SITEr   Z DuplicateEmrFsConfigurationError)rV   r1   configrZ   rZ   r[   rt   Z  s   z-CreateCluster.validate_no_emrfs_configurationN)J__name__
__module____qualname__NAMEr
   ZCREATE_CLUSTER_DESCRIPTIONZDESCRIPTIONZRELEASE_LABELZOS_RELEASE_LABELZAMI_VERSIONr   ZINSTANCE_GROUPS_SCHEMAZINSTANCE_GROUPSZINSTANCE_TYPEZINSTANCE_COUNTZAUTO_TERMINATEZINSTANCE_FLEETS_SCHEMAZINSTANCE_FLEETSZCLUSTER_NAMEZLOG_URIZLOG_ENCRYPTION_KMS_KEY_IDZSERVICE_ROLEZAUTOSCALING_ROLEZUSE_DEFAULT_ROLESZCONFIGURATIONSZEC2_ATTRIBUTESZEC2_ATTRIBUTES_SCHEMAZTERMINATION_PROTECTEDZSCALE_DOWN_BEHAVIORZ
VISIBILITYZ	DEBUGGINGZTAGSZTAGS_SCHEMAZBOOTSTRAP_ACTIONSZBOOTSTRAP_ACTIONS_SCHEMAZAPPLICATIONSZAPPLICATIONS_SCHEMAZEMR_FSZEMR_FS_SCHEMAZSTEPS_SCHEMAZSTEPSZADDITIONAL_INFOZ HBASE_RESTORE_FROM_BACKUP_SCHEMAZRESTORE_FROM_HBASEZSECURITY_CONFIGZCUSTOM_AMI_IDZEBS_ROOT_VOLUME_SIZEZREPO_UPGRADE_ON_BOOTZKERBEROS_ATTRIBUTES_SCHEMAZKERBEROS_ATTRIBUTESZSTEP_CONCURRENCY_LEVELZMANAGED_SCALING_POLICY_SCHEMAZMANAGED_SCALING_POLICYZPLACEMENT_GROUP_CONFIGS_SCHEMAZPLACEMENT_GROUP_CONFIGSZAUTO_TERMINATION_POLICY_SCHEMAZAUTO_TERMINATION_POLICYZ	ARG_TABLEr   Z	FROM_FILEZSYNOPSISZEXAMPLESr\   rU   rH   rP   rJ   rI   r?   rS   rk   rs   rQ   rt   rZ   rZ   rZ   r[   r   !   s   
!#%(+-/2479<?BEHJMOQSUX[^ae vF%r   )rE   Zawscli.customizations.commandsr   Zawscli.customizations.emrr   r   r   r   r   r   r	   r
   r   r   r   Z!awscli.customizations.emr.commandr   Z#awscli.customizations.emr.constantsr   r   Zbotocore.compatr   r   rZ   rZ   rZ   r[   <module>   s$   