diff --git a/awscli/botocore/docs/__init__.py b/awscli/botocore/docs/__init__.py index c6cae1d3a346..e311f4f72862 100644 --- a/awscli/botocore/docs/__init__.py +++ b/awscli/botocore/docs/__init__.py @@ -25,15 +25,21 @@ def generate_docs(root_dir, session): service's reference documentation is loacated at root_dir/reference/services/service-name.rst """ - services_doc_path = os.path.join(root_dir, 'reference', 'services') - if not os.path.exists(services_doc_path): - os.makedirs(services_doc_path) + # Create the root directory where all service docs live. + services_dir_path = os.path.join(root_dir, 'reference', 'services') + if not os.path.exists(services_dir_path): + os.makedirs(services_dir_path) # Generate reference docs and write them out. for service_name in session.get_available_services(): - docs = ServiceDocumenter(service_name, session).document_service() - service_doc_path = os.path.join( - services_doc_path, service_name + '.rst' + docs = ServiceDocumenter( + service_name, session, services_dir_path + ).document_service() + + # Write the main service documentation page. + # Path: /reference/services//index.rst + service_file_path = os.path.join( + services_dir_path, f'{service_name}.rst' ) - with open(service_doc_path, 'wb') as f: + with open(service_file_path, 'wb') as f: f.write(docs) diff --git a/awscli/botocore/docs/bcdoc/restdoc.py b/awscli/botocore/docs/bcdoc/restdoc.py index e7a2823daeaf..ae3abd92b277 100644 --- a/awscli/botocore/docs/bcdoc/restdoc.py +++ b/awscli/botocore/docs/bcdoc/restdoc.py @@ -11,6 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import logging +import os from botocore.compat import OrderedDict from botocore.docs.bcdoc.docstringparser import DocStringParser @@ -219,3 +220,15 @@ def remove_all_sections(self): def clear_text(self): self._writes = [] + + def add_title_section(self, title): + title_section = self.add_new_section('title') + title_section.style.h1(title) + return title_section + + def write_to_file(self, full_path, file_name): + if not os.path.exists(full_path): + os.makedirs(full_path) + sub_resource_file_path = os.path.join(full_path, f'{file_name}.rst') + with open(sub_resource_file_path, 'wb') as f: + f.write(self.flush_structure()) diff --git a/awscli/botocore/docs/client.py b/awscli/botocore/docs/client.py index 0cbf3f9799d2..931e5745dc70 100644 --- a/awscli/botocore/docs/client.py +++ b/awscli/botocore/docs/client.py @@ -11,8 +11,10 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import inspect +import os from botocore.compat import OrderedDict +from botocore.docs.bcdoc.restdoc import DocumentStructure from botocore.docs.example import ResponseExampleDocumenter from botocore.docs.method import ( document_custom_method, @@ -24,9 +26,21 @@ from botocore.docs.utils import DocumentedShape, get_official_service_name +def _allowlist_generate_presigned_url(method_name, service_name, **kwargs): + if method_name != 'generate_presigned_url': + return None + return service_name in ['s3'] + + class ClientDocumenter: - def __init__(self, client, shared_examples=None): + _CLIENT_METHODS_FILTERS = [ + _allowlist_generate_presigned_url, + ] + + def __init__(self, client, root_docs_path, shared_examples=None): self._client = client + self._client_class_name = self._client.__class__.__name__ + self._root_docs_path = root_docs_path self._shared_examples = shared_examples if self._shared_examples is None: self._shared_examples = {} @@ -39,9 +53,32 @@ def document_client(self, section): """ self._add_title(section) self._add_class_signature(section) - client_methods = get_instance_public_methods(self._client) + client_methods = self._get_client_methods() self._add_client_intro(section, client_methods) - self._add_client_methods(section, client_methods) + self._add_client_methods(client_methods) + + def _get_client_methods(self): + client_methods = get_instance_public_methods(self._client) + return self._filter_client_methods(client_methods) + + def _filter_client_methods(self, client_methods): + filtered_methods = {} + for method_name, method in client_methods.items(): + include = self._filter_client_method( + method=method, + method_name=method_name, + service_name=self._service_name, + ) + if include: + filtered_methods[method_name] = method + return filtered_methods + + def _filter_client_method(self, **kwargs): + for filter in self._CLIENT_METHODS_FILTERS: + filter_include = filter(**kwargs) + if filter_include is not None: + return filter_include + return True def _add_title(self, section): section.style.h2('Client') @@ -64,16 +101,16 @@ def _add_client_intro(self, section, client_methods): self._add_client_creation_example(section) # List out all of the possible client methods. - section.style.new_line() - section.write('These are the available methods:') - section.style.new_line() - class_name = self._client.__class__.__name__ + section.style.dedent() + section.style.new_paragraph() + section.writeln('These are the available methods:') + section.style.toctree() for method_name in sorted(client_methods): - section.style.li(f':py:meth:`~{class_name}.Client.{method_name}`') + section.style.tocitem(f'{self._service_name}/client/{method_name}') def _add_class_signature(self, section): section.style.start_sphinx_py_class( - class_name=f'{self._client.__class__.__name__}.Client' + class_name=f'{self._client_class_name}.Client' ) def _add_client_creation_example(self, section): @@ -84,19 +121,29 @@ def _add_client_creation_example(self, section): ) section.style.end_codeblock() - def _add_client_methods(self, section, client_methods): - section = section.add_new_section('methods') + def _add_client_methods(self, client_methods): for method_name in sorted(client_methods): + # Create a new DocumentStructure for each client method and add contents. + method_doc_structure = DocumentStructure( + method_name, target='html' + ) self._add_client_method( - section, method_name, client_methods[method_name] + method_doc_structure, method_name, client_methods[method_name] + ) + # Write client methods in individual/nested files. + # Path: /reference/services//client/.rst + client_dir_path = os.path.join( + self._root_docs_path, self._service_name, 'client' ) + method_doc_structure.write_to_file(client_dir_path, method_name) def _add_client_method(self, section, method_name, method): - section = section.add_new_section(method_name) + section.add_title_section(method_name) + method_section = section.add_new_section(method_name) if self._is_custom_method(method_name): - self._add_custom_method(section, method_name, method) + self._add_custom_method(method_section, method_name, method) else: - self._add_model_driven_method(section, method_name) + self._add_model_driven_method(method_section, method_name) def _is_custom_method(self, method_name): return method_name not in self._client.meta.method_to_api_mapping @@ -109,9 +156,10 @@ def _add_method_exceptions_list(self, section, operation_model): error_section.style.new_line() error_section.style.bold('Exceptions') error_section.style.new_line() - client_name = self._client.__class__.__name__ for error in operation_model.error_shapes: - class_name = f'{client_name}.Client.exceptions.{error.name}' + class_name = ( + f'{self._client_class_name}.Client.exceptions.{error.name}' + ) error_section.style.li(f':py:class:`{class_name}`') def _add_model_driven_method(self, section, method_name): @@ -177,15 +225,18 @@ class ClientExceptionsDocumenter: ), ) - def __init__(self, client): + def __init__(self, client, root_docs_path): self._client = client + self._client_class_name = self._client.__class__.__name__ + self._service_name = self._client.meta.service_model.service_name self._service_id = self._client.meta.service_model.service_id + self._root_docs_path = root_docs_path def document_exceptions(self, section): self._add_title(section) self._add_overview(section) self._add_exceptions_list(section) - self._add_exception_classes(section) + self._add_exception_classes() def _add_title(self, section): section.style.h2('Client Exceptions') @@ -207,7 +258,7 @@ def _add_overview(self, section): def _exception_class_name(self, shape): cls_name = self._client.__class__.__name__ - return f'{cls_name}.Client.exceptions.{shape.name}' + return f'{self._client_class_name}.Client.exceptions.{shape.name}' def _add_exceptions_list(self, section): error_shapes = self._client.meta.service_model.error_shapes @@ -217,17 +268,34 @@ def _add_exceptions_list(self, section): section.style.new_line() return section.style.new_line() - section.write('The available client exceptions are:') - section.style.new_line() + section.writeln('The available client exceptions are:') + section.style.toctree() for shape in error_shapes: - class_name = self._exception_class_name(shape) - section.style.li(f':py:class:`{class_name}`') + section.style.tocitem( + f'{self._service_name}/client/exceptions/{shape.name}' + ) - def _add_exception_classes(self, section): + def _add_exception_classes(self): for shape in self._client.meta.service_model.error_shapes: - self._add_exception_class(section, shape) + # Create a new DocumentStructure for each exception method and add contents. + exception_doc_structure = DocumentStructure( + shape.name, target='html' + ) + self._add_exception_class(exception_doc_structure, shape) + # Write exceptions in individual/nested files. + # Path: /reference/services//client/exceptions/.rst + exception_dir_path = os.path.join( + self._root_docs_path, + self._service_name, + 'client', + 'exceptions', + ) + exception_doc_structure.write_to_file( + exception_dir_path, shape.name + ) def _add_exception_class(self, section, shape): + section.add_title_section(shape.name) class_section = section.add_new_section(shape.name) class_name = self._exception_class_name(shape) class_section.style.start_sphinx_py_class(class_name=class_name) diff --git a/awscli/botocore/docs/paginator.py b/awscli/botocore/docs/paginator.py index 9deebba69262..b77afb349266 100644 --- a/awscli/botocore/docs/paginator.py +++ b/awscli/botocore/docs/paginator.py @@ -10,18 +10,27 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import os + from botocore import xform_name from botocore.compat import OrderedDict +from botocore.docs.bcdoc.restdoc import DocumentStructure from botocore.docs.method import document_model_driven_method from botocore.docs.utils import DocumentedShape from botocore.utils import get_service_module_name class PaginatorDocumenter: - def __init__(self, client, service_paginator_model): + def __init__(self, client, service_paginator_model, root_docs_path): self._client = client + self._client_class_name = self._client.__class__.__name__ self._service_name = self._client.meta.service_model.service_name self._service_paginator_model = service_paginator_model + self._root_docs_path = root_docs_path + self._USER_GUIDE_LINK = ( + 'https://boto3.amazonaws.com/' + 'v1/documentation/api/latest/guide/paginators.html' + ) def document_paginators(self, section): """Documents the various paginators for a service @@ -29,8 +38,10 @@ def document_paginators(self, section): param section: The section to write to. """ section.style.h2('Paginators') + self._add_overview(section) section.style.new_line() section.writeln('The available paginators are:') + section.style.toctree() paginator_names = sorted( self._service_paginator_model._paginator_config @@ -38,39 +49,67 @@ def document_paginators(self, section): # List the available paginators and then document each paginator. for paginator_name in paginator_names: - section.style.li( - f':py:class:`{self._client.__class__.__name__}.Paginator.{paginator_name}`' + section.style.tocitem( + f'{self._service_name}/paginator/{paginator_name}' + ) + # Create a new DocumentStructure for each paginator and add contents. + paginator_doc_structure = DocumentStructure( + paginator_name, target='html' + ) + self._add_paginator(paginator_doc_structure, paginator_name) + # Write paginators in individual/nested files. + # Path: /reference/services//paginator/.rst + paginator_dir_path = os.path.join( + self._root_docs_path, self._service_name, 'paginator' + ) + paginator_doc_structure.write_to_file( + paginator_dir_path, paginator_name ) - self._add_paginator(section, paginator_name) def _add_paginator(self, section, paginator_name): - section = section.add_new_section(paginator_name) + section.add_title_section(paginator_name) # Docment the paginator class - section.style.start_sphinx_py_class( - class_name=f'{self._client.__class__.__name__}.Paginator.{paginator_name}' + paginator_section = section.add_new_section(paginator_name) + paginator_section.style.start_sphinx_py_class( + class_name=f'{self._client_class_name}.Paginator.{paginator_name}' ) - section.style.start_codeblock() - section.style.new_line() + paginator_section.style.start_codeblock() + paginator_section.style.new_line() # Document how to instantiate the paginator. - section.write( + paginator_section.write( f'paginator = client.get_paginator(\'{xform_name(paginator_name)}\')' ) - section.style.end_codeblock() - section.style.new_line() + paginator_section.style.end_codeblock() + paginator_section.style.new_line() # Get the pagination model for the particular paginator. paginator_config = self._service_paginator_model.get_paginator( paginator_name ) document_paginate_method( - section=section, + section=paginator_section, paginator_name=paginator_name, event_emitter=self._client.meta.events, service_model=self._client.meta.service_model, paginator_config=paginator_config, ) + def _add_overview(self, section): + section.style.new_line() + section.write( + 'Paginators are available on a client instance ' + 'via the ``get_paginator`` method. For more detailed instructions ' + 'and examples on the usage of paginators, see the ' + 'paginators ' + ) + section.style.external_link( + title='user guide', + link=self._USER_GUIDE_LINK, + ) + section.write('.') + section.style.new_line() + def document_paginate_method( section, diff --git a/awscli/botocore/docs/service.py b/awscli/botocore/docs/service.py index 33a8e3d7da9b..b17aa2ef34f9 100644 --- a/awscli/botocore/docs/service.py +++ b/awscli/botocore/docs/service.py @@ -19,9 +19,10 @@ class ServiceDocumenter: - def __init__(self, service_name, session): + def __init__(self, service_name, session, root_docs_path): self._session = session self._service_name = service_name + self._root_docs_path = root_docs_path self._client = self._session.create_client( service_name, @@ -73,10 +74,14 @@ def client_api(self, section): except DataNotFoundError: pass - ClientDocumenter(self._client, examples).document_client(section) + ClientDocumenter( + self._client, self._root_docs_path, examples + ).document_client(section) def client_exceptions(self, section): - ClientExceptionsDocumenter(self._client).document_exceptions(section) + ClientExceptionsDocumenter( + self._client, self._root_docs_path + ).document_exceptions(section) def paginator_api(self, section): try: @@ -85,10 +90,11 @@ def paginator_api(self, section): ) except DataNotFoundError: return - paginator_documenter = PaginatorDocumenter( - self._client, service_paginator_model - ) - paginator_documenter.document_paginators(section) + if service_paginator_model._paginator_config: + paginator_documenter = PaginatorDocumenter( + self._client, service_paginator_model, self._root_docs_path + ) + paginator_documenter.document_paginators(section) def waiter_api(self, section): if self._client.waiter_names: @@ -96,7 +102,7 @@ def waiter_api(self, section): self._service_name ) waiter_documenter = WaiterDocumenter( - self._client, service_waiter_model + self._client, service_waiter_model, self._root_docs_path ) waiter_documenter.document_waiters(section) diff --git a/awscli/botocore/docs/waiter.py b/awscli/botocore/docs/waiter.py index 672642523e77..c4d73a8f1cff 100644 --- a/awscli/botocore/docs/waiter.py +++ b/awscli/botocore/docs/waiter.py @@ -10,18 +10,27 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import os + from botocore import xform_name from botocore.compat import OrderedDict +from botocore.docs.bcdoc.restdoc import DocumentStructure from botocore.docs.method import document_model_driven_method from botocore.docs.utils import DocumentedShape from botocore.utils import get_service_module_name class WaiterDocumenter: - def __init__(self, client, service_waiter_model): + def __init__(self, client, service_waiter_model, root_docs_path): self._client = client + self._client_class_name = self._client.__class__.__name__ self._service_name = self._client.meta.service_model.service_name self._service_waiter_model = service_waiter_model + self._root_docs_path = root_docs_path + self._USER_GUIDE_LINK = ( + 'https://boto3.amazonaws.com/' + 'v1/documentation/api/latest/guide/clients.html#waiters' + ) def document_waiters(self, section): """Documents the various waiters for a service. @@ -29,38 +38,64 @@ def document_waiters(self, section): :param section: The section to write to. """ section.style.h2('Waiters') + self._add_overview(section) section.style.new_line() section.writeln('The available waiters are:') + section.style.toctree() for waiter_name in self._service_waiter_model.waiter_names: - section.style.li( - f':py:class:`{self._client.__class__.__name__}.Waiter.{waiter_name}`' + section.style.tocitem(f'{self._service_name}/waiter/{waiter_name}') + # Create a new DocumentStructure for each waiter and add contents. + waiter_doc_structure = DocumentStructure( + waiter_name, target='html' + ) + self._add_single_waiter(waiter_doc_structure, waiter_name) + # Write waiters in individual/nested files. + # Path: /reference/services//waiter/.rst + waiter_dir_path = os.path.join( + self._root_docs_path, self._service_name, 'waiter' ) - self._add_single_waiter(section, waiter_name) + waiter_doc_structure.write_to_file(waiter_dir_path, waiter_name) def _add_single_waiter(self, section, waiter_name): - section = section.add_new_section(waiter_name) - section.style.start_sphinx_py_class( - class_name=f'{self._client.__class__.__name__}.Waiter.{waiter_name}' + section.add_title_section(waiter_name) + waiter_section = section.add_new_section(waiter_name) + waiter_section.style.start_sphinx_py_class( + class_name=f"{self._client_class_name}.Waiter.{waiter_name}" ) # Add example on how to instantiate waiter. - section.style.start_codeblock() - section.style.new_line() - section.write( + waiter_section.style.start_codeblock() + waiter_section.style.new_line() + waiter_section.write( f'waiter = client.get_waiter(\'{xform_name(waiter_name)}\')' ) - section.style.end_codeblock() + waiter_section.style.end_codeblock() # Add information on the wait() method - section.style.new_line() + waiter_section.style.new_line() document_wait_method( - section=section, + section=waiter_section, waiter_name=waiter_name, event_emitter=self._client.meta.events, service_model=self._client.meta.service_model, service_waiter_model=self._service_waiter_model, ) + def _add_overview(self, section): + section.style.new_line() + section.write( + 'Waiters are available on a client instance ' + 'via the ``get_waiter`` method. For more detailed instructions ' + 'and examples on the usage or waiters, see the ' + 'waiters ' + ) + section.style.external_link( + title='user guide', + link=self._USER_GUIDE_LINK, + ) + section.write('.') + section.style.new_line() + def document_wait_method( section, diff --git a/tests/functional/botocore/docs/__init__.py b/tests/functional/botocore/docs/__init__.py index faf9e4531e34..af3520ccaa07 100644 --- a/tests/functional/botocore/docs/__init__.py +++ b/tests/functional/botocore/docs/__init__.py @@ -10,6 +10,10 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import os +import shutil +import tempfile + from botocore.docs.service import ServiceDocumenter from botocore.session import get_session @@ -19,6 +23,13 @@ class BaseDocsFunctionalTest(unittest.TestCase): def setUp(self): self._session = get_session() + self.docs_root_dir = tempfile.mkdtemp() + self.root_services_path = os.path.join( + self.docs_root_dir, 'reference', 'services' + ) + + def tearDown(self): + shutil.rmtree(self.docs_root_dir) def assert_contains_line(self, line, contents): contents = contents.decode('utf-8') @@ -40,9 +51,21 @@ def assert_not_contains_lines(self, lines, contents): for line in lines: self.assertNotIn(line, contents) + def get_client_method_contents(self, service_name, method_name): + service_file_path = os.path.join( + self.root_services_path, + service_name, + 'client', + f'{method_name}.rst', + ) + with open(service_file_path, 'rb') as f: + return f.read() + def get_title_section_for(self, service_name): contents = ( - ServiceDocumenter(service_name, self._session) + ServiceDocumenter( + service_name, self._session, self.root_services_path + ) .document_service() .decode('utf-8') ) @@ -54,48 +77,49 @@ def get_title_section_for(self, service_name): def get_method_document_block(self, operation_name, contents): contents = contents.decode('utf-8') - start_method_document = f' .. py:method:: {operation_name}(' + start_method_document = '.. py:method:: %s(' % operation_name start_index = contents.find(start_method_document) self.assertNotEqual(start_index, -1, 'Method is not found in contents') contents = contents[start_index:] - end_index = contents.find( - ' .. py:method::', len(start_method_document) - ) + end_index = contents.find('.. py:method::', len(start_method_document)) contents = contents[:end_index] return contents.encode('utf-8') def get_parameter_document_block(self, param_name, contents): contents = contents.decode('utf-8') - start_param_document = f' :type {param_name}:' + start_param_document = ' :type %s:' % param_name start_index = contents.find(start_param_document) self.assertNotEqual(start_index, -1, 'Param is not found in contents') contents = contents[start_index:] - end_index = contents.find(' :type', len(start_param_document)) + end_index = contents.find(' :type', len(start_param_document)) contents = contents[:end_index] return contents.encode('utf-8') def get_parameter_documentation_from_service( self, service_name, method_name, param_name ): - contents = ServiceDocumenter( - service_name, self._session + ServiceDocumenter( + service_name, self._session, self.root_services_path ).document_service() + contents = self.get_client_method_contents(service_name, method_name) method_contents = self.get_method_document_block(method_name, contents) return self.get_parameter_document_block(param_name, method_contents) def get_docstring_for_method(self, service_name, method_name): - contents = ServiceDocumenter( - service_name, self._session + ServiceDocumenter( + service_name, self._session, self.root_services_path ).document_service() + contents = self.get_client_method_contents(service_name, method_name) method_contents = self.get_method_document_block(method_name, contents) return method_contents def assert_is_documented_as_autopopulated_param( self, service_name, method_name, param_name, doc_string=None ): - contents = ServiceDocumenter( - service_name, self._session + ServiceDocumenter( + service_name, self._session, self.root_services_path ).document_service() + contents = self.get_client_method_contents(service_name, method_name) method_contents = self.get_method_document_block(method_name, contents) # Ensure it is not in the example. diff --git a/tests/functional/botocore/docs/test_s3.py b/tests/functional/botocore/docs/test_s3.py index 1e4fb1c46391..259c84e02667 100644 --- a/tests/functional/botocore/docs/test_s3.py +++ b/tests/functional/botocore/docs/test_s3.py @@ -46,17 +46,22 @@ def test_hides_content_md5_when_impossible_to_provide(self): 'put_object_acl', 'put_bucket_versioning', ] - service_contents = ServiceDocumenter( - 's3', self._session + ServiceDocumenter( + 's3', self._session, self.root_services_path ).document_service() for method_name in modified_methods: + contents = self.get_client_method_contents('s3', method_name) method_contents = self.get_method_document_block( - method_name, service_contents + method_name, contents ) self.assertNotIn( 'ContentMD5=\'string\'', method_contents.decode('utf-8') ) + def test_generate_presigned_url_documented(self): + content = self.get_docstring_for_method('s3', 'generate_presigned_url') + self.assert_contains_line('generate_presigned_url', content) + def test_copy_source_documented_as_union_type(self): content = self.get_docstring_for_method('s3', 'copy_object') dict_form = ( diff --git a/tests/functional/botocore/docs/test_secretsmanager.py b/tests/functional/botocore/docs/test_secretsmanager.py new file mode 100644 index 000000000000..8a4e42f29444 --- /dev/null +++ b/tests/functional/botocore/docs/test_secretsmanager.py @@ -0,0 +1,24 @@ +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from botocore.docs.service import ServiceDocumenter + +from tests.functional.botocore.docs import BaseDocsFunctionalTest + + +class TestSecretsManagerDocs(BaseDocsFunctionalTest): + def test_generate_presigned_url_is_not_documented(self): + documenter = ServiceDocumenter( + 'secretsmanager', self._session, self.root_services_path + ) + docs = documenter.document_service() + self.assert_not_contains_line('generate_presigned_url', docs) diff --git a/tests/functional/botocore/docs/test_streaming_body.py b/tests/functional/botocore/docs/test_streaming_body.py index da570baaf66f..6cd41d6b9d9a 100644 --- a/tests/functional/botocore/docs/test_streaming_body.py +++ b/tests/functional/botocore/docs/test_streaming_body.py @@ -34,8 +34,9 @@ def test_all_streaming_body_are_properly_documented(self): ) def assert_streaming_body_is_properly_documented(self, service, operation): - service_docs = ServiceDocumenter( - service, self._session + ServiceDocumenter( + service, self._session, self.root_services_path ).document_service() - method_docs = self.get_method_document_block(operation, service_docs) + contents = self.get_client_method_contents(service, operation) + method_docs = self.get_method_document_block(operation, contents) self.assert_contains_line('StreamingBody', method_docs) diff --git a/tests/unit/botocore/docs/__init__.py b/tests/unit/botocore/docs/__init__.py index eab84bdd2f9e..275598eadb67 100644 --- a/tests/unit/botocore/docs/__init__.py +++ b/tests/unit/botocore/docs/__init__.py @@ -43,6 +43,10 @@ def setUp(self): self.example_model_file = os.path.join( self.version_dirs, 'examples-1.json' ) + self.docs_root_dir = tempfile.mkdtemp() + self.root_services_path = os.path.join( + self.docs_root_dir, 'reference', 'services' + ) self.json_model = {} self.nested_json_model = {} @@ -55,6 +59,7 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.root_dir) + shutil.rmtree(self.docs_root_dir) def setup_client(self): with open(self.example_model_file, 'w') as f: @@ -237,8 +242,16 @@ def assert_contains_line(self, line): contents = self.doc_structure.flush_structure().decode('utf-8') self.assertIn(line, contents) - def assert_contains_lines_in_order(self, lines): - contents = self.doc_structure.flush_structure().decode('utf-8') + def get_nested_service_contents(self, service, type, name): + service_file_path = os.path.join( + self.root_services_path, service, type, f'{name}.rst' + ) + with open(service_file_path, 'rb') as f: + return f.read().decode('utf-8') + + def assert_contains_lines_in_order(self, lines, contents=None): + if contents is None: + contents = self.doc_structure.flush_structure().decode('utf-8') for line in lines: self.assertIn(line, contents) beginning = contents.find(line) diff --git a/tests/unit/botocore/docs/test_client.py b/tests/unit/botocore/docs/test_client.py index d51d55d42651..7664a5337c10 100644 --- a/tests/unit/botocore/docs/test_client.py +++ b/tests/unit/botocore/docs/test_client.py @@ -29,7 +29,9 @@ def setUp(self): self.add_shape_to_params('Biz', 'String') self.add_shape_to_errors('SomeException') self.setup_client() - self.client_documenter = ClientDocumenter(self.client) + self.client_documenter = ClientDocumenter( + self.client, self.root_services_path + ) def test_document_client(self): self.client_documenter.document_client(self.doc_structure) @@ -42,42 +44,66 @@ def test_document_client(self): ' A low-level client representing AWS MyService', ' AWS MyService Description', ' client = session.create_client(\'myservice\')', - ' These are the available methods:', - ' * :py:meth:`~MyService.Client.can_paginate`', - ' * :py:meth:`~MyService.Client.get_paginator`', - ' * :py:meth:`~MyService.Client.get_waiter`', - ' * :py:meth:`~MyService.Client.sample_operation`', - ' .. py:method:: can_paginate(operation_name)', - ' .. py:method:: get_paginator(operation_name)', - ' .. py:method:: get_waiter(waiter_name)', - ' .. py:method:: sample_operation(**kwargs)', - ' **Request Syntax**', + 'These are the available methods:', + ' myservice/client/can_paginate', + ' myservice/client/get_paginator', + ' myservice/client/get_waiter', + ' myservice/client/sample_operation', + ] + ) + self.assert_contains_lines_in_order( + ['.. py:method:: can_paginate(operation_name)'], + self.get_nested_service_contents( + 'myservice', 'client', 'can_paginate' + ), + ) + self.assert_contains_lines_in_order( + ['.. py:method:: get_paginator(operation_name)'], + self.get_nested_service_contents( + 'myservice', 'client', 'get_paginator' + ), + ) + self.assert_contains_lines_in_order( + ['.. py:method:: get_waiter(waiter_name)'], + self.get_nested_service_contents( + 'myservice', 'client', 'get_waiter' + ), + ) + self.assert_contains_lines_in_order( + [ + '.. py:method:: sample_operation(**kwargs)', + ' **Request Syntax**', + ' ::', + ' response = client.sample_operation(', + ' Biz=\'string\'', + ' )', + ' :type Biz: string', + ' :param Biz:', + ' :rtype: dict', + ' :returns:', + ' **Response Syntax**', ' ::', - ' response = client.sample_operation(', - ' Biz=\'string\'', - ' )', - ' :type Biz: string', - ' :param Biz:', - ' :rtype: dict', - ' :returns:', - ' **Response Syntax**', - ' ::', - ' {', - ' \'Biz\': \'string\'', - ' }', - ' **Response Structure**', - ' - *(dict) --*', - ' - **Biz** *(string) --*', + ' {', + ' \'Biz\': \'string\'', + ' }', + ' **Response Structure**', + ' - *(dict) --*', + ' - **Biz** *(string) --*', '**Exceptions**', - '* :py:class:`MyService.Client.exceptions.SomeException`', - ] + '* :py:class:`MyService.Client.exceptions.SomeException`', + ], + self.get_nested_service_contents( + 'myservice', 'client', 'sample_operation' + ), ) class TestClientExceptionsDocumenter(BaseDocsTest): def setup_documenter(self): self.setup_client() - self.exceptions_documenter = ClientExceptionsDocumenter(self.client) + self.exceptions_documenter = ClientExceptionsDocumenter( + self.client, self.root_services_path + ) def test_no_modeled_exceptions(self): self.setup_documenter() @@ -110,7 +136,14 @@ def test_modeled_exceptions(self): '=================', 'Client exceptions are available', 'The available client exceptions are:', - '* :py:class:`MyService.Client.exceptions.SomeException`', + '.. toctree::', + ':maxdepth: 1', + ':titlesonly:', + ' myservice/client/exceptions/SomeException', + ] + ) + self.assert_contains_lines_in_order( + [ '.. py:class:: MyService.Client.exceptions.SomeException', '**Example** ::', 'except client.exceptions.SomeException as e:', @@ -129,5 +162,8 @@ def test_modeled_exceptions(self): '- **Error** *(dict) --* ', '- **Code** *(string) --* ', '- **Message** *(string) --* ', - ] + ], + self.get_nested_service_contents( + 'myservice', 'client/exceptions', 'SomeException' + ), ) diff --git a/tests/unit/botocore/docs/test_docs.py b/tests/unit/botocore/docs/test_docs.py index c7d147424f22..71085687e253 100644 --- a/tests/unit/botocore/docs/test_docs.py +++ b/tests/unit/botocore/docs/test_docs.py @@ -11,8 +11,6 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import os -import shutil -import tempfile from botocore.docs import generate_docs from botocore.session import get_session @@ -24,7 +22,6 @@ class TestGenerateDocs(BaseDocsTest): def setUp(self): super().setUp() - self.docs_root = tempfile.mkdtemp() self.loader_patch = mock.patch( 'botocore.session.create_loader', return_value=self.loader ) @@ -37,20 +34,16 @@ def setUp(self): def tearDown(self): super().tearDown() - shutil.rmtree(self.docs_root) self.loader_patch.stop() self.available_service_patch.stop() def test_generate_docs(self): session = get_session() # Have the rst files get written to the temporary directory - generate_docs(self.docs_root, session) + generate_docs(self.docs_root_dir, session) - reference_services_path = os.path.join( - self.docs_root, 'reference', 'services' - ) reference_service_path = os.path.join( - reference_services_path, 'myservice.rst' + self.root_services_path, 'myservice.rst' ) self.assertTrue(os.path.exists(reference_service_path)) diff --git a/tests/unit/botocore/docs/test_paginator.py b/tests/unit/botocore/docs/test_paginator.py index 1a5f6a7ab2bc..06cb8f42d28e 100644 --- a/tests/unit/botocore/docs/test_paginator.py +++ b/tests/unit/botocore/docs/test_paginator.py @@ -26,7 +26,9 @@ def extra_setup(self): self.setup_client() paginator_model = PaginatorModel(self.paginator_json_model) self.paginator_documenter = PaginatorDocumenter( - client=self.client, service_paginator_model=paginator_model + client=self.client, + service_paginator_model=paginator_model, + root_docs_path=self.root_services_path, ) def test_document_paginators(self): @@ -37,7 +39,11 @@ def test_document_paginators(self): 'Paginators', '==========', 'The available paginators are:', - '* :py:class:`MyService.Paginator.SampleOperation`', + 'paginator/SampleOperation', + ] + ) + self.assert_contains_lines_in_order( + [ '.. py:class:: MyService.Paginator.SampleOperation', ' ::', ' paginator = client.get_paginator(\'sample_operation\')', @@ -79,7 +85,10 @@ def test_document_paginators(self): ' - *(dict) --*', ' - **Biz** *(string) --*', ' - **NextToken** *(string) --*', - ] + ], + self.get_nested_service_contents( + 'myservice', 'paginator', 'SampleOperation' + ), ) def test_no_page_size_if_no_limit_key(self): diff --git a/tests/unit/botocore/docs/test_service.py b/tests/unit/botocore/docs/test_service.py index 9227b43751f9..0e1f9670d3cc 100644 --- a/tests/unit/botocore/docs/test_service.py +++ b/tests/unit/botocore/docs/test_service.py @@ -28,7 +28,9 @@ def setUp(self): 'botocore.session.create_loader', return_value=self.loader ): session = get_session() - self.service_documenter = ServiceDocumenter('myservice', session) + self.service_documenter = ServiceDocumenter( + 'myservice', session, self.root_services_path + ) def test_document_service(self): # Note that not everything will be included as it is just @@ -47,31 +49,48 @@ def test_document_service(self): ' A low-level client representing AWS MyService', ' AWS MyService Description', ' client = session.create_client(\'myservice\')', - ' These are the available methods:', - ' * :py:meth:`~MyService.Client.sample_operation`', - ' .. py:method:: sample_operation(**kwargs)', - ' **Examples** ', - ' Sample Description.', - ' ::', - ' response = client.sample_operation(', + 'These are the available methods:', + ' myservice/client/sample_operation', '=================', 'Client Exceptions', '=================', + 'Client exceptions are available on a client instance ', + 'via the ``exceptions`` property. For more detailed instructions ', + 'and examples on the exact usage of client exceptions, see the ', + 'error handling ', 'Client exceptions are available', '==========', 'Paginators', '==========', - '.. py:class:: MyService.Paginator.SampleOperation', - ' .. py:method:: paginate(**kwargs)', + 'Paginators are available on a client instance', + 'via the ``get_paginator`` method. For more detailed instructions ', + 'and examples on the usage of paginators, see the paginators', + 'The available paginators are:', + ' myservice/paginator/SampleOperation', '=======', 'Waiters', '=======', - '.. py:class:: MyService.Waiter.SampleOperationComplete', - ' .. py:method:: wait(**kwargs)', + 'Waiters are available on a client instance ', + 'via the ``get_waiter`` method. For more detailed instructions ', + 'and examples on the usage or waiters, see the waiters', + ' myservice/waiter/SampleOperationComplete', ] for line in lines: self.assertIn(line, contents) + self.assert_contains_lines_in_order( + [ + '.. py:method:: sample_operation(**kwargs)', + ' **Examples** ', + ' Sample Description.', + ' ::', + ' response = client.sample_operation(', + ], + self.get_nested_service_contents( + 'myservice', 'client', 'sample_operation' + ), + ) + def test_document_service_no_paginator(self): os.remove(self.paginator_model_file) contents = self.service_documenter.document_service().decode('utf-8') diff --git a/tests/unit/botocore/docs/test_waiter.py b/tests/unit/botocore/docs/test_waiter.py index de51f9397b4d..815e34399558 100644 --- a/tests/unit/botocore/docs/test_waiter.py +++ b/tests/unit/botocore/docs/test_waiter.py @@ -23,7 +23,9 @@ def setUp(self): self.setup_client() waiter_model = WaiterModel(self.waiter_json_model) self.waiter_documenter = WaiterDocumenter( - client=self.client, service_waiter_model=waiter_model + client=self.client, + service_waiter_model=waiter_model, + root_docs_path=self.root_services_path, ) def test_document_waiters(self): @@ -34,7 +36,11 @@ def test_document_waiters(self): 'Waiters', '=======', 'The available waiters are:', - '* :py:class:`MyService.Waiter.SampleOperationComplete`', + 'waiter/SampleOperationComplete', + ] + ) + self.assert_contains_lines_in_order( + [ '.. py:class:: MyService.Waiter.SampleOperationComplete', ' ::', ' waiter = client.get_waiter(\'sample_operation_complete\')', @@ -65,5 +71,8 @@ def test_document_waiters(self): ' - **MaxAttempts** *(integer) --*', ' The maximum number of attempts to be made. Default: 40', ' :returns: None', - ] + ], + self.get_nested_service_contents( + 'myservice', 'waiter', 'SampleOperationComplete' + ), )