diff --git a/.gitmodules b/.gitmodules index 5467b79e5c..64e0013615 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "src/deps/Imath"] path = src/deps/Imath url = https://github.com/AcademySoftwareFoundation/Imath +[submodule "src/deps/minizip-ng"] + path = src/deps/minizip-ng + url = https://github.com/zlib-ng/minizip-ng.git diff --git a/CMakeLists.txt b/CMakeLists.txt index faf69c35e7..a833c0cae7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,7 @@ option(OTIO_INSTALL_COMMANDLINE_TOOLS "Install the OTIO command line tools" ON) option(OTIO_FIND_IMATH "Find Imath using find_package" OFF) option(OTIO_FIND_PYBIND11 "Find pybind11 using find_package" OFF) option(OTIO_FIND_RAPIDJSON "Find RapidJSON using find_package" OFF) +option(OTIO_FIND_MINIZIP_NG "Find minizip-ng using find_package" OFF) set(OTIO_PYTHON_INSTALL_DIR "" CACHE STRING "Python installation dir (such as the site-packages dir)") # Build options @@ -270,7 +271,6 @@ else() endif() #----- RapidJSON - if(OTIO_FIND_RAPIDJSON) find_package(RapidJSON CONFIG REQUIRED) if (RapidJSON_FOUND) @@ -280,6 +280,16 @@ else() message(STATUS "Using src/deps/rapidjson by default") endif() +#----- minizip-ng +if(OTIO_FIND_MINIZIP_NG) + find_package(minizip-ng REQUIRED) + if (minizip-ng_FOUND) + message(STATUS "Found minizip-ng at ${minizip-ng_CONFIG}") + endif() +else() + message(STATUS "Using src/deps/minizip-ng by default") +endif() + # set up the internally hosted dependencies add_subdirectory(src/deps) diff --git a/docs/tutorials/otio-plugins.md b/docs/tutorials/otio-plugins.md index 7a8d9c478f..120eeca03e 100644 --- a/docs/tutorials/otio-plugins.md +++ b/docs/tutorials/otio-plugins.md @@ -146,6 +146,7 @@ into a single directory named with a suffix of .otiod. - write_to_file: - input_otio - filepath + - relative_media_base_dir - media_policy - dryrun @@ -182,6 +183,7 @@ read on unix and windows platforms. - write_to_file: - input_otio - filepath + - relative_media_base_dir - media_policy - dryrun diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 9a07779301..2e8568a158 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -4,14 +4,17 @@ include_directories(${PROJECT_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}/../src ${PYTHON_INCLUDE_DIRS}) -list(APPEND examples conform) -list(APPEND examples flatten_video_tracks) -list(APPEND examples summarize_timing) -list(APPEND examples io_perf_test) -list(APPEND examples upgrade_downgrade_example) +set(examples + bundle + conform + flatten_video_tracks + summarize_timing + io_perf_test + upgrade_downgrade_example) if(OTIO_PYTHON_INSTALL) - list(APPEND examples python_adapters_child_process) - list(APPEND examples python_adapters_embed) + list(APPEND examples + python_adapters_child_process + python_adapters_embed) endif() foreach(example ${examples}) add_executable(${example} ${example}.cpp util.h util.cpp) diff --git a/examples/bundle.cpp b/examples/bundle.cpp new file mode 100644 index 0000000000..b7ef262352 --- /dev/null +++ b/examples/bundle.cpp @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project + +// Example for converting an .otio file into a bundle. + +#include "util.h" + +#include + +#include + +using namespace OTIO_NS; + +int +main(int argc, char** argv) +{ + if (argc != 3) { + std::cout << "Usage: bundle (input.otio) (output.otioz|output.otiod)" << std::endl; + return 1; + } + const std::string input = examples::normalize_path(argv[1]); + const std::string output = examples::normalize_path(argv[2]); + + // Read the timeline + ErrorStatus error_status; + SerializableObject::Retainer timeline(dynamic_cast( + Timeline::from_json_file(input, &error_status))); + if (!timeline || is_error(error_status)) { + examples::print_error(error_status); + return 1; + } + + // Write the bundle + bundle::WriteOptions options; + options.relative_media_base_dir = + std::filesystem::u8path(input).parent_path().u8string(); + auto const ext = std::filesystem::u8path(output).extension().u8string(); + if (".otiod" == ext) + { + if (!bundle::write_otiod( + timeline, + output, + options, + &error_status)) { + examples::print_error(error_status); + return 1; + } + } + else + { + if (!bundle::write_otioz( + timeline, + output, + options, + &error_status)) { + examples::print_error(error_status); + return 1; + } + } + + return 0; +} diff --git a/src/deps/CMakeLists.txt b/src/deps/CMakeLists.txt index 97e41d8abc..1af2a96e0a 100644 --- a/src/deps/CMakeLists.txt +++ b/src/deps/CMakeLists.txt @@ -17,6 +17,10 @@ if(NOT OTIO_FIND_RAPIDJSON) set(DEPS_SUBMODULES ${DEPS_SUBMODULES} rapidjson) endif() +if(NOT OTIO_FIND_MINIZIP_NG) + set(DEPS_SUBMODULES ${DEPS_SUBMODULES} minizip-ng) +endif() + foreach(submodule IN LISTS DEPS_SUBMODULES) file(GLOB SUBMOD_CONTENTS "${submodule}/*") list(LENGTH SUBMOD_CONTENTS SUBMOD_CONTENT_LEN) @@ -60,3 +64,19 @@ if(NOT OTIO_FIND_IMATH) endif() endif() +if(NOT OTIO_FIND_MINIZIP_NG) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + set(MZ_BZIP2 OFF) + set(MZ_LZMA OFF) + set(MZ_PPMD OFF) + set(MZ_ZSTD OFF) + set(MZ_LIBCOMP OFF) + set(MZ_PKCRYPT OFF) + set(MZ_WZAES OFF) + set(MZ_OPENSSL OFF) + set(MZ_LIBBSD OFF) + set(MZ_ICONV OFF) + set(MZ_FETCH_LIBS ON) + set(SKIP_INSTALL_ALL ON) + add_subdirectory(minizip-ng) +endif() diff --git a/src/deps/minizip-ng b/src/deps/minizip-ng new file mode 160000 index 0000000000..d69cb0a539 --- /dev/null +++ b/src/deps/minizip-ng @@ -0,0 +1 @@ +Subproject commit d69cb0a5392332d6a55e9b56405a6fa2fc8b157d diff --git a/src/opentimelineio/CMakeLists.txt b/src/opentimelineio/CMakeLists.txt index 607cb6dec2..7dd26ec79c 100644 --- a/src/opentimelineio/CMakeLists.txt +++ b/src/opentimelineio/CMakeLists.txt @@ -4,6 +4,7 @@ set(OPENTIMELINEIO_HEADER_FILES anyDictionary.h anyVector.h + bundle.h color.h clip.h composable.h @@ -41,6 +42,7 @@ set(OPENTIMELINEIO_HEADER_FILES version.h) add_library(opentimelineio ${OTIO_SHARED_OR_STATIC_LIB} + bundle.cpp color.cpp clip.cpp composable.cpp @@ -90,9 +92,9 @@ else() PRIVATE "${PROJECT_SOURCE_DIR}/src/deps/rapidjson/include") endif() - -target_link_libraries(opentimelineio - PUBLIC opentime Imath::Imath) +target_link_libraries(opentimelineio + PUBLIC opentime Imath::Imath + PRIVATE MINIZIP::minizip) set_target_properties(opentimelineio PROPERTIES DEBUG_POSTFIX "${OTIO_DEBUG_POSTFIX}" @@ -134,6 +136,19 @@ if(OTIO_CXX_INSTALL) set(OPENTIMELINEIO_INCLUDES ${OTIO_RESOLVED_CXX_INSTALL_DIR}/include) + if(NOT OTIO_FIND_MINIZIP_NG) + install(TARGETS minizip + EXPORT OpenTimelineIOTargets + ARCHIVE DESTINATION "${OTIO_RESOLVED_CXX_DYLIB_INSTALL_DIR}" + LIBRARY DESTINATION "${OTIO_RESOLVED_CXX_DYLIB_INSTALL_DIR}" + RUNTIME DESTINATION "${OTIO_RESOLVED_CXX_DYLIB_INSTALL_DIR}") + endif() + + set(OTIO_CONFIG_DEPENDENCIES "") + if(OTIO_FIND_MINIZIP_NG) + string(APPEND OTIO_CONFIG_DEPENDENCIES "find_dependency(minizip-ng)\n") + endif() + install(TARGETS opentimelineio EXPORT OpenTimelineIOTargets INCLUDES DESTINATION "${OPENTIMELINEIO_INCLUDES}" diff --git a/src/opentimelineio/OpenTimelineIOConfig.cmake.in b/src/opentimelineio/OpenTimelineIOConfig.cmake.in index 355f8ea952..58a3abb2e7 100644 --- a/src/opentimelineio/OpenTimelineIOConfig.cmake.in +++ b/src/opentimelineio/OpenTimelineIOConfig.cmake.in @@ -3,5 +3,6 @@ include(CMakeFindDependencyMacro) find_dependency(OpenTime) find_dependency(Imath) +@OTIO_CONFIG_DEPENDENCIES@ include("${CMAKE_CURRENT_LIST_DIR}/OpenTimelineIOTargets.cmake") diff --git a/src/opentimelineio/bundle.cpp b/src/opentimelineio/bundle.cpp new file mode 100644 index 0000000000..1338da6a90 --- /dev/null +++ b/src/opentimelineio/bundle.cpp @@ -0,0 +1,953 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project + +#include "opentimelineio/bundle.h" + +#include "opentimelineio/clip.h" +#include "opentimelineio/externalReference.h" +#include "opentimelineio/imageSequenceReference.h" +#include "opentimelineio/missingReference.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace opentimelineio { namespace OPENTIMELINEIO_VERSION_NS { +namespace bundle { + + // File contents: + // - URL parsing utilities (percent_decode, starts_with, to_lower, file_from_url) + // - BundleFile and bundle file processing + // - ZipWriter and ZipReader + // - Public API (dry_run, write_otioz, read_otioz, write_otiod, read_otiod) + + namespace { + + // Utility to decode a URL (ie, %20 -> ' ') + std::string percent_decode(std::string const& s) + { + std::string result; + result.reserve(s.size()); + for (size_t i = 0; i < s.size(); ++i) + { + if (s[i] == '%' && i + 2 < s.size()) + { + int hi = std::stoi(s.substr(i + 1, 2), nullptr, 16); + result += static_cast(hi); + i += 2; + } + else + { + result += s[i]; + } + } + return result; + } + + // Utility to check if a string has the given prefix, case insensitive + bool starts_with(std::string_view s, std::string_view prefix) + { + if (s.size() < prefix.size()) return false; + for (size_t i = 0; i < prefix.size(); ++i) + { + if (std::tolower(static_cast(s[i])) != prefix[i]) return false; + } + return true; + } + + // Utility to lowercase a string + std::string to_lower(std::string const& s) + { + std::string out = s; + std::transform(out.begin(), out.end(), out.begin(), + [](unsigned char c){ return static_cast(std::tolower(c)); }); + return out; + } + + // This struct contains the source path and archive name for each + // file to be added to the bundle + struct BundleFile + { + std::string source_path; + std::string archive_name; + + bool operator<(BundleFile const& other) const + { + return std::tie(source_path, archive_name) < + std::tie(other.source_path, other.archive_name); + } + }; + using BundleFiles = std::set; + + // Check and add files to the bundle. This is a convenience function + // used by process_media_references(). + bool register_bundle_file( + std::filesystem::path const& source_path, + std::filesystem::path const& relative_media_base_dir, + std::map& paths, + BundleFiles& out, + ErrorStatus* error_status) + { + // Check if this file would overwrite any others + auto const paths_key = to_lower(source_path.filename().u8string()); + auto const i = paths.find(paths_key); + if (i != paths.end() && + source_path.parent_path() != i->second.parent_path()) + { + if (error_status) + *error_status = ErrorStatus( + ErrorStatus::FILE_WRITE_FAILED, + "media file '" + source_path.u8string() + + "' would overwrite '" + i->second.u8string() + "'"); + return false; + } + paths[paths_key] = source_path; + + // Add the file to the bundle list + auto resolved = source_path; + if (resolved.is_relative() && !relative_media_base_dir.empty()) + resolved = relative_media_base_dir / resolved; + auto const bundle_path = (std::filesystem::u8path(media_dir) / + source_path.filename()).u8string(); + out.insert({ resolved.u8string(), bundle_path }); + return true; + } + + // Utility to create a missing reference + MissingReference* create_missing_reference( + MediaReference* ref, + std::string const& reason, + std::optional const& original_target_url) + { + auto out = new MissingReference( + ref->name(), + std::nullopt, + ref->metadata()); + out->metadata()["missing_reference_because"] = reason; + if (original_target_url.has_value()) + { + out->metadata()["original_target_url"] = *original_target_url; + } + return out; + } + + // Process all media references according to the policy. The list of + // files to be added to the bundle is returned. + bool process_media_references( + Timeline* timeline, + std::filesystem::path const& relative_media_base_dir, + MediaReferencePolicy policy, + ErrorStatus* error_status, + BundleFiles& out) + { + // Store a map of the processed paths to check we are not + // overwriting any media in the bundle + std::map paths; + + // Iterate over all the clips + for (auto clip : timeline->find_clips()) + { + // Get the media reference retainers + std::map> refs; + for (auto i : clip->media_references()) + { + refs[i.first] = i.second; + } + + // Iterate over the media references + bool modified = false; + for (auto& ref : refs) + { + std::optional file; + std::optional original_target_url; + if (auto ext = dynamic_retainer_cast(ref.second)) + { + // Handle external references + file = file_from_url(ext->target_url()); + if (file.has_value() && + policy != MediaReferencePolicy::all_missing) + { + auto const path = std::filesystem::u8path(*file); + if (!register_bundle_file( + path, + relative_media_base_dir, + paths, + out, + error_status)) + return false; + + // Set the URL to the bundle location + ext->set_target_url((std::filesystem::u8path(media_dir) / + path.filename()).u8string()); + } + + // Save the URL for the missing reference metadata + original_target_url = ext->target_url(); + } + else if (auto seq = dynamic_retainer_cast(ref.second)) + { + // Handle image sequence references + if (policy != MediaReferencePolicy::all_missing) + { + for (int frame = 0; + frame < seq->number_of_images_in_sequence(); + frame += seq->frame_step()) + { + file = file_from_url(seq->target_url_for_image_number(frame)); + if (file.has_value()) + { + const auto path = std::filesystem::u8path(*file); + if (!register_bundle_file( + path, + relative_media_base_dir, + paths, + out, + error_status)) + return false; + } + } + + // Set the URL to the bundle location + seq->set_target_url_base(std::string(media_dir) + "/"); + } + + // Save the URL for the missing reference metadata + original_target_url = file_from_url(seq->target_url_for_image_number(0)); + } + + // Handle the policy for this reference + if (ref.second && + !dynamic_retainer_cast(ref.second)) + { + switch (policy) + { + case MediaReferencePolicy::error_if_not_file: + if (!file.has_value()) + { + if (error_status) + *error_status = ErrorStatus( + ErrorStatus::FILE_WRITE_FAILED, + "media reference '" + + ref.second->name() + "' is not a file"); + return false; + } + break; + case MediaReferencePolicy::missing_if_not_file: + if (!file.has_value()) + { + ref.second = create_missing_reference( + ref.second, + "'missing_if_not_file' specified as the MediaReferencePolicy", + original_target_url); + modified = true; + } + break; + case MediaReferencePolicy::all_missing: + { + ref.second = create_missing_reference( + ref.second, + "'all_missing' specified as the MediaReferencePolicy", + original_target_url); + modified = true; + break; + } + default: break; + } + } + } + + if (modified) + { + // Set the new media references on the clip + Clip::MediaReferences ref_ptrs; + for (auto i : refs) + { + ref_ptrs[i.first] = i.second; + } + clip->set_media_references( + ref_ptrs, + clip->active_media_reference_key()); + } + } + return true; + } + + // Common code for preparing the bundle + bool prepare_bundle( + Timeline const* timeline, + WriteOptions const& options, + ErrorStatus* error_status, + SerializableObject::Retainer& out_clone, + std::string& out_json, + BundleFiles& out_files) + { + // Clone the timeline so we can make modifications + out_clone = SerializableObject::Retainer( + dynamic_cast(timeline->clone(error_status))); + if (!out_clone || is_error(error_status)) + return false; + + // Get the relative media path + std::filesystem::path relative_media_base_dir; + if (options.relative_media_base_dir.has_value()) + relative_media_base_dir = std::filesystem::u8path( + *options.relative_media_base_dir); + + // Process the media references and get the list of files to add + // to the bundle + if (!process_media_references( + out_clone, + relative_media_base_dir, + options.policy, + error_status, + out_files)) + return false; + + // Convert the new timeline to json for writing + out_json = out_clone->to_json_string( + error_status, + nullptr, + options.indent); + return !is_error(error_status); + } + + // Change relative paths to absolute + void rewrite_media_to_absolute( + Timeline* timeline, + std::filesystem::path const& bundle_root) + { + for (auto clip : timeline->find_clips()) + { + for (auto ref : clip->media_references()) + { + if (auto ext = dynamic_cast(ref.second)) + { + auto const current = std::filesystem::u8path(ext->target_url()); + if (current.is_relative()) + { + ext->set_target_url( + (bundle_root / current).lexically_normal().u8string()); + } + } + else if (auto seq = dynamic_cast(ref.second)) + { + auto const base = std::filesystem::u8path(seq->target_url_base()); + if (base.is_relative()) + { + auto absolute = (bundle_root / base).lexically_normal().u8string(); + if (!absolute.empty() && absolute.back() != '/') + absolute += '/'; + seq->set_target_url_base(absolute); + } + } + } + } + } + + // This class writes a ZIP file using minizip-ng + class ZipWriter + { + public: + ZipWriter(std::string const& path) : + _path(path) + { + _writer = mz_zip_writer_create(); + if (!_writer) + throw std::runtime_error( + "cannot create zip writer for '" + path + "'"); + if (mz_zip_writer_open_file(_writer, path.c_str(), 0, 0) != MZ_OK) + { + mz_zip_writer_delete(&_writer); + throw std::runtime_error( + "cannot initialize zip writer for '" + path + "'"); + } + } + + ~ZipWriter() + { + if (_writer) + { + if (!_finalized) + mz_zip_writer_close(_writer); + mz_zip_writer_delete(&_writer); + } + if (!_finalized) + { + std::error_code ec; + std::filesystem::remove(_path, ec); + } + } + + ZipWriter(ZipWriter const&) = delete; + ZipWriter& operator=(ZipWriter const&) = delete; + + void add_text(std::string const& name, std::string const& text) + { + if (text.size() > static_cast(INT32_MAX)) + throw std::runtime_error( + "text entry '" + name + "' too large for zip '" + + _path + "'"); + + mz_zip_file file_info = {}; + file_info.filename = name.c_str(); + file_info.modified_date = time(nullptr); + file_info.version_madeby = MZ_VERSION_MADEBY; + file_info.compression_method = MZ_COMPRESS_METHOD_DEFLATE; + file_info.flag = MZ_ZIP_FLAG_UTF8; + + if (mz_zip_writer_add_buffer( + _writer, + const_cast(text.data()), + static_cast(text.size()), + &file_info) != MZ_OK) + { + throw std::runtime_error( + "cannot add '" + name + "' to zip '" + _path + "'"); + } + } + + void add_file_uncompressed(std::string const& name, std::string const& path) + { + mz_zip_writer_set_compress_method(_writer, MZ_COMPRESS_METHOD_STORE); + int32_t const err = mz_zip_writer_add_file(_writer, path.c_str(), name.c_str()); + mz_zip_writer_set_compress_method(_writer, MZ_COMPRESS_METHOD_DEFLATE); + if (err != MZ_OK) + throw std::runtime_error( + "cannot add '" + path + "' to zip '" + _path + "'"); + } + + void finalize() + { + if (_finalized) return; + if (mz_zip_writer_close(_writer) != MZ_OK) + throw std::runtime_error( + "cannot finalize zip writer for '" + _path + "'"); + _finalized = true; + } + + private: + std::string _path; + void* _writer = nullptr; + bool _finalized = false; + }; + + // This class reads a ZIP file using minizip-ng + class ZipReader + { + public: + ZipReader(std::string const& path) + { + _path = path; + _reader = mz_zip_reader_create(); + if (!_reader) + throw std::runtime_error( + "cannot create zip reader for '" + path + "'"); + if (mz_zip_reader_open_file(_reader, path.c_str()) != MZ_OK) + { + mz_zip_reader_delete(&_reader); + throw std::runtime_error( + "cannot open zip file '" + path + "'"); + } + } + + ~ZipReader() + { + if (_reader) + mz_zip_reader_delete(&_reader); + } + + ZipReader(ZipReader const&) = delete; + ZipReader& operator=(ZipReader const&) = delete; + + // Read a named entry to a string (used for content.otio). Returns nullopt + // if the entry is not present. + std::optional read_to_string(std::string const& name) + { + if (mz_zip_reader_locate_entry(_reader, name.c_str(), 0) != MZ_OK) + return std::nullopt; + if (mz_zip_reader_entry_open(_reader) != MZ_OK) + throw std::runtime_error( + "cannot open zip entry '" + name + "' in '" + + _path + "'"); + + // Close the entry on any exit path. + struct EntryScope { + void* r; ~EntryScope() { mz_zip_reader_entry_close(r); } + } scope{ _reader }; + + mz_zip_file* info = nullptr; + if (mz_zip_reader_entry_get_info(_reader, &info) != MZ_OK || !info) + throw std::runtime_error( + "cannot stat zip entry '" + name + "' in '" + + _path + "'"); + if (info->uncompressed_size < 0 || + info->uncompressed_size > static_cast(INT32_MAX)) + throw std::runtime_error( + "zip entry '" + name + "' too large in '" + + _path + "'"); + + std::string out(static_cast(info->uncompressed_size), '\0'); + int32_t const n = mz_zip_reader_entry_read( + _reader, out.data(), static_cast(out.size())); + if (n != static_cast(out.size())) + throw std::runtime_error( + "cannot extract zip '" + name + + "' entry to memory in '" + _path + "'"); + return out; + } + + // Iterate every entry, invoking fn(filename, is_dir) for each. fn may call + // extract_current() to save the current entry to disk. + template + void for_each_entry(Fn&& fn) + { + int32_t err = mz_zip_reader_goto_first_entry(_reader); + if (err == MZ_END_OF_LIST) + return; // empty archive + if (err != MZ_OK) + throw std::runtime_error( + "cannot read zip entry in '" + _path + "'"); + while (err == MZ_OK) + { + mz_zip_file* info = nullptr; + if (mz_zip_reader_entry_get_info(_reader, &info) != MZ_OK || !info) + throw std::runtime_error( + "cannot stat zip entry in '" + _path + "'"); + + bool const is_dir = (mz_zip_reader_entry_is_dir(_reader) == MZ_OK); + fn(std::string(info->filename ? info->filename : ""), is_dir); + + err = mz_zip_reader_goto_next_entry(_reader); + if (err != MZ_OK && err != MZ_END_OF_LIST) + throw std::runtime_error( + "cannot advance zip entry in '" + _path + "'"); + } + } + + // Save the entry the cursor is currently on to a file. Call only from + // within a for_each_entry callback. + void extract_current_to_file(std::string const& path) + { + if (mz_zip_reader_entry_save_file(_reader, path.c_str()) != MZ_OK) + throw std::runtime_error("cannot extract zip entry in '" + + _path + "' to '" + path + "'"); + } + + private: + std::string _path; + void* _reader = nullptr; + }; + + // Validate that an extraction path is contained within the destination + // directory. Protects against zip slip vulnerabilities where archive + // entries contain ".." or absolute paths. + bool is_path_safe( + std::filesystem::path const& dest_dir, + std::filesystem::path const& out_path) + { + auto const canonical_dest = std::filesystem::weakly_canonical(dest_dir); + auto const canonical_out = std::filesystem::weakly_canonical(out_path); + auto const rel = std::filesystem::relative(canonical_out, canonical_dest); + if (rel.empty()) return false; + auto const first = rel.begin()->u8string(); + return first != ".."; + } + } + + std::optional file_from_url(std::string const& url) + { + constexpr std::string_view file_prefix = "file://"; + if (!starts_with(url, file_prefix)) + { + if (url.find("://") != std::string::npos) return std::nullopt; + return url; // bare path + } + + // Split "file://" + authority + path + std::string rest = std::string(url.substr(file_prefix.size())); + auto const slash = rest.find('/'); + std::string netloc = (slash == std::string::npos) ? rest : rest.substr(0, slash); + std::string path = (slash == std::string::npos) ? "" : rest.substr(slash); + + // Strip query/fragment from path + if (auto pos = path.find('?'); pos != std::string::npos) path.resize(pos); + if (auto pos = path.find('#'); pos != std::string::npos) path.resize(pos); + + // Decode the path + path = percent_decode(path); + + auto is_drive = [](std::string const& s) { + return s.size() == 2 && + std::isalpha(static_cast(s[0])) && + s[1] == ':'; + }; + + std::string result; + + if (is_drive(netloc)) + { + // file://X:/path → X:/path + result = netloc + path; + } + else if (path.size() >= 3 && path[0] == '/' && is_drive(path.substr(1, 2))) + { + // file://host/X:/path → X:/path (strip leading '/' and host) + result = path.substr(1); + } + else if (!netloc.empty() && to_lower(netloc) != "localhost") + { + // file://host/path → //host/path (UNC) + result = "//" + netloc + path; + } + else + { + // file:///path or file://localhost/path → /path + result = path; + } + + // Normalize separators + std::replace(result.begin(), result.end(), '\\', '/'); + return result; + } + + std::optional dry_run( + Timeline const* timeline, + WriteOptions const& options, + ErrorStatus* error_status) + { + // Prepare the bundle + SerializableObject::Retainer clone; + std::string json; + BundleFiles files; + if (!prepare_bundle( + timeline, + options, + error_status, + clone, + json, + files)) + return std::nullopt; + + // Add the version file and timeline file sizes + uint64_t total = 0; + total += std::string_view(version).size(); + total += json.size(); + + // Add the media file sizes + try + { + for (auto const& f : files) + { + total += std::filesystem::file_size( + std::filesystem::u8path(f.source_path)); + } + } + catch (std::exception const& e) + { + if (error_status) + *error_status = ErrorStatus( + ErrorStatus::FILE_OPEN_FAILED, + e.what()); + return std::nullopt; + } + + return total; + } + + bool write_otioz( + Timeline const* timeline, + std::string const& path, + WriteOptions const& options, + ErrorStatus* error_status) + { + // Validate the path + if (std::filesystem::exists(std::filesystem::u8path(path))) + { + if (error_status) + *error_status = ErrorStatus( + ErrorStatus::FILE_WRITE_FAILED, + "output path '" + path + "' already exists"); + return false; + } + + // Prepare the bundle + SerializableObject::Retainer clone; + std::string json; + BundleFiles files; + if (!prepare_bundle( + timeline, + options, + error_status, + clone, + json, + files)) + return false; + + // Write the bundle + try + { + ZipWriter zw(path); + zw.add_text(version_file, version); + zw.add_text(timeline_file, json); + for (const auto& i : files) + { + zw.add_file_uncompressed(i.archive_name, i.source_path); + } + zw.finalize(); + } + catch (const std::exception& e) + { + if (error_status) + { + *error_status = ErrorStatus( + ErrorStatus::FILE_WRITE_FAILED, + "error writing '" + path + "': " + e.what()); + } + return false; + } + return true; + } + + SerializableObject* read_otioz( + std::string const& path, + ReadOptions const& options, + ErrorStatus* error_status) + { + // Validate the paths + auto const input_path = std::filesystem::u8path(path); + if (!std::filesystem::is_regular_file(input_path)) + { + if (error_status) + *error_status = ErrorStatus( + ErrorStatus::FILE_OPEN_FAILED, + "input '" + path + "' is not a file"); + return nullptr; + } + std::filesystem::path output_path; + if (options.extract_path.has_value()) + { + output_path = options.extract_path.value(); + if (std::filesystem::exists(output_path)) + { + if (error_status) + *error_status = ErrorStatus( + ErrorStatus::FILE_WRITE_FAILED, + "output directory '" + output_path.u8string() + + "' already exists"); + return nullptr; + } + } + + // Read the archive + std::string json; + try + { + ZipReader zr(path); + + // Read the timeline + auto json_opt = zr.read_to_string(timeline_file); + if (!json_opt) + throw std::runtime_error( + "'" + path + "' is missing content.otio"); + json = *json_opt; + + // Extract the archive + if (options.extract_path.has_value()) + { + std::filesystem::create_directories(output_path); + + zr.for_each_entry([&](std::string const& filename, bool is_dir) + { + auto const file_path = output_path / std::filesystem::u8path(filename); + + // Guard against zip slip + if (!is_path_safe(output_path, file_path)) + throw std::runtime_error( + "unsafe path '" + filename + "' in '" + + path + "'"); + + if (is_dir) + { + std::filesystem::create_directories(file_path); + } + else + { + std::filesystem::create_directories(file_path.parent_path()); + zr.extract_current_to_file(file_path.u8string()); + } + }); + } + } + catch (const std::exception& e) + { + if (error_status) + *error_status = ErrorStatus( + ErrorStatus::FILE_OPEN_FAILED, + "error reading '" + path + "': " + e.what()); + + if (options.extract_path.has_value()) + { + // Try and remove the directory if it was partially written + std::error_code ec; + std::filesystem::remove_all(output_path, ec); + } + + return nullptr; + } + + // Create the timeline + auto result = Timeline::from_json_string(json, error_status); + if (!result || is_error(error_status)) + return nullptr; + + // Optionally make the media reference paths absolute + if (options.absolute_media_reference_paths && + options.extract_path) + { + if (auto timeline = dynamic_cast(result)) + { + rewrite_media_to_absolute( + timeline, + std::filesystem::u8path(options.extract_path.value())); + } + } + + return result; + } + + bool write_otiod( + Timeline const* timeline, + std::string const& path, + WriteOptions const& options, + ErrorStatus* error_status) + { + // Validate the path + auto const output_path = std::filesystem::u8path(path); + if (std::filesystem::exists(output_path)) + { + if (error_status) + *error_status = ErrorStatus( + ErrorStatus::FILE_WRITE_FAILED, + "output path '" + path + "' already exists"); + return false; + } + + // Prepare the bundle + SerializableObject::Retainer clone; + std::string json; + BundleFiles files; + if (!prepare_bundle( + timeline, + options, + error_status, + clone, + json, + files)) + return false; + + // Write the bundle + try + { + // Create the output directories + std::filesystem::create_directories(output_path); + std::filesystem::create_directory(output_path / media_dir); + + // Write the version file + { + std::ofstream of(output_path / version_file); + of.exceptions(std::ios::failbit | std::ios::badbit); + of << version; + } + + // Write the timeline + { + std::ofstream of(output_path / timeline_file); + of.exceptions(std::ios::failbit | std::ios::badbit); + of << json; + } + + // Copy the media files + for (const auto& i : files) + { + std::filesystem::copy_file( + i.source_path, + output_path / std::filesystem::u8path(i.archive_name)); + } + } + catch (const std::exception& e) + { + if (error_status) + { + *error_status = ErrorStatus( + ErrorStatus::FILE_WRITE_FAILED, + "error writing '" + path + "': " + e.what()); + } + return false; + } + return true; + } + + SerializableObject* read_otiod( + std::string const& path, + ReadOptions const& options, + ErrorStatus* error_status) + { + // Validate the paths + auto const input_path = std::filesystem::u8path(path); + if (!std::filesystem::is_directory(input_path)) + { + if (error_status) + *error_status = ErrorStatus( + ErrorStatus::FILE_OPEN_FAILED, + "input '" + path + "' is not a directory"); + return nullptr; + } + auto const version_path = input_path / version_file; + if (!std::filesystem::is_regular_file(version_path)) + { + if (error_status) + *error_status = ErrorStatus( + ErrorStatus::FILE_OPEN_FAILED, + "'" + path + "' is missing a version file"); + return nullptr; + } + auto const timeline_path = input_path / timeline_file; + if (!std::filesystem::is_regular_file(timeline_path)) + { + if (error_status) + *error_status = ErrorStatus( + ErrorStatus::FILE_OPEN_FAILED, + "'" + path + "' is missing a timeline file"); + return nullptr; + } + + // Read the timeline + auto result = Timeline::from_json_file( + timeline_path.u8string(), + error_status); + if (!result || is_error(error_status)) + return nullptr; + + // Optionally make the media reference paths absolute + if (options.absolute_media_reference_paths) + { + if (auto timeline = dynamic_cast(result)) + { + rewrite_media_to_absolute( + timeline, + timeline_path.parent_path()); + } + } + + return result; + } +} +} +} + diff --git a/src/opentimelineio/bundle.h b/src/opentimelineio/bundle.h new file mode 100644 index 0000000000..daa46a7a36 --- /dev/null +++ b/src/opentimelineio/bundle.h @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project + +#pragma once + +#include "opentimelineio/timeline.h" + +namespace opentimelineio { namespace OPENTIMELINEIO_VERSION_NS { + +/// @brief Utilities for working with OTIO bundles (otioz and otiod) +/// +/// https://opentimelineio.readthedocs.io/en/stable/tutorials/otio-filebundles.html#otioz-d-file-bundle-format-details +namespace bundle { + + /// @brief This constant provides the bundle file version. + static char constexpr version[] = "1.0.0"; + + /// @brief This constant provides the name of the bundled version file. + static char constexpr version_file[] = "version.txt"; + + /// @brief This constant provides the name of the bundled timeline file. + static char constexpr timeline_file[] = "content.otio"; + + /// @brief This constant provides the name of the bundled media sub-directory. + static char constexpr media_dir[] = "media"; + + /// @brief This enumeration provides the media reference policy. + /// + /// Note that the policy is not applied to missing references. For + /// example, if you have a timeline with missing references, those do not + /// count as errors when the policy is set to error_if_not_file. + enum MediaReferencePolicy + { + error_if_not_file, + missing_if_not_file, + all_missing + }; + + /// @brief Get a file from a URL. + std::optional file_from_url(std::string const& url); + + /// @brief Options for writing bundles. + struct OTIO_API_TYPE WriteOptions + { + /// @brief Base directory for resolving relative media reference paths. + /// If a media reference URL resolves to a relative path, it is resolved + /// against this directory before being added to the bundle. + std::optional relative_media_base_dir; + + /// @brief Media reference policy. + MediaReferencePolicy policy = MediaReferencePolicy::error_if_not_file; + + /// @brief Number of spaces for JSON indentation. + int indent = 4; + }; + + /// @brief Options for reading bundles. + struct OTIO_API_TYPE ReadOptions + { + /// @brief Extract the contents of the otioz bundle to this directory, + /// which must not already exist. If this is not set then only the timeline + /// is read from the bundle. + std::optional extract_path; + + /// @brief Convert the media reference paths to absolute paths. + /// + /// If this is set to true for otioz files, an extract_path must also be set. + bool absolute_media_reference_paths = false; + }; + + /// @brief Check the timeline against the error policy to see if a bundle + /// can be made correctly. If so, return the total uncompressed size of + /// the files that would be written to a bundle, without actually writing it. + /// This is useful for estimating the disk space required. + OTIO_API std::optional dry_run( + Timeline const* timeline, + WriteOptions const& options = WriteOptions(), + ErrorStatus* error_status = nullptr); + + /// @brief Write a timeline and it's referenced media to an otioz bundle. + OTIO_API bool write_otioz( + Timeline const* timeline, + std::string const& path, + WriteOptions const& options = WriteOptions(), + ErrorStatus* error_status = nullptr); + + /// @brief Read a timeline from an otioz bundle. + /// + /// The default behavior is to only read the timeline from the bundle. + /// To also extract the contents, set extract_path in the read options. + OTIO_API SerializableObject* read_otioz( + std::string const& path, + ReadOptions const& options = ReadOptions(), + ErrorStatus* error_status = nullptr); + + /// @brief Write a timeline and it's referenced media to an otiod bundle. + OTIO_API bool write_otiod( + Timeline const* timeline, + std::string const& path, + WriteOptions const& options = WriteOptions(), + ErrorStatus* error_status = nullptr); + + /// @brief Read a timeline from an otiod bundle. + OTIO_API SerializableObject* read_otiod( + std::string const& path, + ReadOptions const& options = ReadOptions(), + ErrorStatus* error_status = nullptr); +} +} +} + diff --git a/src/py-opentimelineio/opentimelineio-bindings/CMakeLists.txt b/src/py-opentimelineio/opentimelineio-bindings/CMakeLists.txt index d37da3629f..5362e62655 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/CMakeLists.txt +++ b/src/py-opentimelineio/opentimelineio-bindings/CMakeLists.txt @@ -15,7 +15,8 @@ pybind11_add_module(_otio otio_imath.cpp otio_tests.cpp otio_serializableObjects.cpp - otio_utils.cpp + otio_utils.cpp + otio_bundle.cpp ${_OTIO_HEADER_FILES}) target_include_directories(_otio diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index f8f8bb6c34..071c88eab1 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -193,6 +193,7 @@ PYBIND11_MODULE(_otio, m) otio_imath_bindings(m); otio_serializable_object_bindings(m); otio_tests_bindings(m); + otio_bundle_bindings(m); m.def( "_serialize_json_to_string", diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.h b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.h index dc5287076b..cd03a9ae51 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.h +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.h @@ -11,3 +11,4 @@ void otio_any_vector_bindings(pybind11::module); void otio_imath_bindings(pybind11::module); void otio_serializable_object_bindings(pybind11::module); void otio_tests_bindings(pybind11::module); +void otio_bundle_bindings(pybind11::module); diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bundle.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bundle.cpp new file mode 100644 index 0000000000..8a1784bd38 --- /dev/null +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bundle.cpp @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project + +#include +#include +#include + +#include "otio_errorStatusHandler.h" + +#include + +using namespace OTIO_NS; +using namespace OTIO_NS::bundle; + +namespace py = pybind11; + +void otio_bundle_bindings(pybind11::module m) +{ + auto mbundle = m.def_submodule("bundle"); + + py::enum_(mbundle, "MediaReferencePolicy", +R"docstring( +This enumeration provides the bundle media reference policy. +)docstring") + .value( + "error_if_not_file", + MediaReferencePolicy::error_if_not_file, + "Return an error if there are any non-file media references.") + .value( + "missing_if_not_file", + MediaReferencePolicy::missing_if_not_file, + "Replace non-file media references with missing references.") + .value( + "all_missing", + MediaReferencePolicy::all_missing, + "Replace all media references with missing references."); + + py::class_(mbundle, "WriteOptions", +R"docstring( +Options for writing bundles. +)docstring") + .def(py::init<>()) + .def_readwrite( + "relative_media_base_dir", + &WriteOptions::relative_media_base_dir, + "Base directory for resolving relative media reference paths. " + "If a media reference URL resolves to a relative path, it is resolved " + "against this directory before being added to the bundle.") + .def_readwrite( + "policy", + &WriteOptions::policy, + "The media reference policy.") + .def_readwrite( + "indent", + &WriteOptions::indent, + "Number of spaces for JSON indentation."); + + py::class_(mbundle, "ReadOptions", +R"docstring( +Options for reading bundles. +)docstring") + .def(py::init<>()) + .def_readwrite( + "extract_path", + &ReadOptions::extract_path, + "Extract the contents of the otioz bundle to this directory, " + "which must not already exist.") + .def_readwrite( + "absolute_media_reference_paths", + &ReadOptions::absolute_media_reference_paths, + "Convert the media reference paths to absolute paths. " + "If this is set to true for otioz files, an extract_path must also be set."); + + mbundle.def( + "dry_run", + []( + Timeline const* timeline, + WriteOptions const& options = WriteOptions()) + { + return dry_run(timeline, options, ErrorStatusHandler()); + }, + "Calculate the total uncompressed size of the files that would be " + "written to a bundle, without actually writing it. This is useful for " + "estimating the disk space required.", + py::arg("timeline"), + py::arg("options") = WriteOptions()); + + mbundle.def( + "write_otioz", + []( + Timeline const* timeline, + std::string const& path, + WriteOptions const& options = WriteOptions()) + { + return write_otioz(timeline, path, options, ErrorStatusHandler()); + }, + "Write a timeline and it's referenced media to an .otioz bundle.", + py::arg("timeline"), + py::arg("path"), + py::arg("options") = WriteOptions()); + + mbundle.def( + "read_otioz", + []( + std::string const& path, + ReadOptions const& options = ReadOptions()) + { + return read_otioz(path, options, ErrorStatusHandler()); + }, + "Read a timeline from an .otioz bundle.", + py::arg("path"), + py::arg("options") = ReadOptions()); + + mbundle.def( + "write_otiod", + []( + Timeline const* timeline, + std::string const& path, + WriteOptions const& options = WriteOptions()) + { + return write_otiod(timeline, path, options, ErrorStatusHandler()); + }, + "Write a timeline and it's referenced media to an .otiod bundle.", + py::arg("timeline"), + py::arg("path"), + py::arg("options") = WriteOptions()); + + mbundle.def( + "read_otiod", + [](std::string const& path, + ReadOptions const& options = ReadOptions()) + { + return read_otiod(path, options, ErrorStatusHandler()); + }, + "Read a timeline from an .otiod bundle.", + py::arg("path"), + py::arg("options") = ReadOptions()); +} diff --git a/src/py-opentimelineio/opentimelineio/__init__.py b/src/py-opentimelineio/opentimelineio/__init__.py index 036907f4f1..869ac1f9b9 100644 --- a/src/py-opentimelineio/opentimelineio/__init__.py +++ b/src/py-opentimelineio/opentimelineio/__init__.py @@ -22,6 +22,5 @@ adapters, hooks, algorithms, - url_utils, versioning, ) diff --git a/src/py-opentimelineio/opentimelineio/adapters/__init__.py b/src/py-opentimelineio/opentimelineio/adapters/__init__.py index 7ee6557742..642336cec2 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/__init__.py +++ b/src/py-opentimelineio/opentimelineio/adapters/__init__.py @@ -28,13 +28,11 @@ # OTIO Json, OTIOZ and OTIOD adapters are always available from . import ( # noqa: F401 otio_json, # core JSON adapter - file_bundle_utils, # utilities for working with OTIO file bundles ) __all__ = [ 'Adapter', 'otio_json', - 'file_bundle_utils', 'suffixes_with_defined_adapters', 'available_adapter_names', 'from_filepath', diff --git a/src/py-opentimelineio/opentimelineio/adapters/file_bundle_utils.py b/src/py-opentimelineio/opentimelineio/adapters/file_bundle_utils.py deleted file mode 100644 index 818299cafa..0000000000 --- a/src/py-opentimelineio/opentimelineio/adapters/file_bundle_utils.py +++ /dev/null @@ -1,172 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright Contributors to the OpenTimelineIO project - -"""Common utilities used by the file bundle adapters (otiod and otioz).""" - -import os -import copy - -from .. import ( - exceptions, - schema, - url_utils, -) - -import urllib - - -# versioning -BUNDLE_VERSION = "1.0.0" -BUNDLE_VERSION_FILE = "version.txt" - -# other variables -BUNDLE_PLAYLIST_PATH = "content.otio" -BUNDLE_DIR_NAME = "media" - - -class NotAFileOnDisk(exceptions.OTIOError): - pass - - -class MediaReferencePolicy: - ErrorIfNotFile = "ErrorIfNotFile" - MissingIfNotFile = "MissingIfNotFile" - AllMissing = "AllMissing" - - -def reference_cloned_and_missing(orig_mr, reason_missing): - """Replace orig_mr with a missing reference with the same metadata. - - Also adds original_target_url and missing_reference_because fields. - """ - - orig_mr = copy.deepcopy(orig_mr) - media_reference = schema.MissingReference() - media_reference.__dict__ = orig_mr.__dict__ - media_reference.metadata['missing_reference_because'] = reason_missing - media_reference.metadata['original_target_url'] = orig_mr.target_url - - return media_reference - - -def _guarantee_unique_basenames(path_list, adapter_name): - # walking across all unique file references, guarantee that all the - # basenames are unique - basename_to_source_fn = {} - for fn in path_list: - new_basename = os.path.basename(fn) - if new_basename in basename_to_source_fn: - raise exceptions.OTIOError( - f"Error: the {adapter_name} adapter requires that the media" - f" files have unique basenames. File '{fn}' and" - f" '{basename_to_source_fn[new_basename]}' have matching" - f" basenames of: '{new_basename}'" - ) - basename_to_source_fn[new_basename] = fn - - -def _prepped_otio_for_bundle_and_manifest( - input_otio, # otio to process - media_policy, # how to handle media references (see: MediaReferencePolicy) - adapter_name, # just for error messages -): - """ Create a new OTIO based on input_otio that has had media references - replaced according to the media_policy. Return that new OTIO and a - mapping of all the absolute file paths (not URLs) to be used in the bundle, - mapped to MediaReferences associated with those files. Media references in - the OTIO will be relinked by the adapters to point to their output - locations. - - The otio[dz] adapters use this function to do further relinking and build - their bundles. - - This is considered an internal API. - - media_policy is expected to be of type MediaReferencePolicy. - """ - - # make sure the incoming OTIO isn't edited - result_otio = copy.deepcopy(input_otio) - - path_to_reference_map = {} - invalid_files = set() - - # result_otio is manipulated in place - for cl in result_otio.find_clips(): - if media_policy == MediaReferencePolicy.AllMissing: - cl.media_reference = reference_cloned_and_missing( - cl.media_reference, - f"{media_policy} specified as the MediaReferencePolicy" - ) - continue - - try: - target_url = cl.media_reference.target_url - except AttributeError: - # not an ExternalReference, ignoring it. - continue - - parsed_url = urllib.parse.urlparse(target_url) - - # ensure that the urlscheme is either "file" or "" - # file means "absolute path" - # "" is interpreted as a relative path, relative to cwd of the python - # process - if parsed_url.scheme not in ("file", ""): - if media_policy is MediaReferencePolicy.ErrorIfNotFile: - raise NotAFileOnDisk( - f"The {adapter_name} adapter only works with media" - " reference target_url attributes that begin with 'file:'." - f" Got a target_url of: '{target_url}'" - ) - if media_policy is MediaReferencePolicy.MissingIfNotFile: - cl.media_reference = reference_cloned_and_missing( - cl.media_reference, - "target_url is not a file scheme url (start with url:)" - ) - continue - - # get an absolute path to the target file - target_file = os.path.abspath(url_utils.filepath_from_url(target_url)) - - # if the file hasn't already been checked - if ( - target_file not in path_to_reference_map - and target_file not in invalid_files - and ( - not os.path.exists(target_file) - or not os.path.isfile(target_file) - ) - ): - invalid_files.add(target_file) - - if target_file in invalid_files: - if media_policy is MediaReferencePolicy.ErrorIfNotFile: - raise NotAFileOnDisk(target_file) - if media_policy is MediaReferencePolicy.MissingIfNotFile: - cl.media_reference = reference_cloned_and_missing( - cl.media_reference, - "target_url target is not a file or does not exist" - ) - - # do not need to relink it in the future or add this target to - # the manifest, because the path is either not a file or does - # not exist. - continue - - # add the media reference to the list of references that point at this - # file, they will need to be relinked - path_to_reference_map.setdefault(target_file, []).append( - cl.media_reference - ) - - _guarantee_unique_basenames(path_to_reference_map.keys(), adapter_name) - - return result_otio, path_to_reference_map - - -def _total_file_size_of(filepaths): - fsize = 0 - for fn in filepaths: - fsize += os.path.getsize(fn) - return fsize diff --git a/src/py-opentimelineio/opentimelineio/adapters/otiod.py b/src/py-opentimelineio/opentimelineio/adapters/otiod.py index f00f4a74a4..71a9c98994 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otiod.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otiod.py @@ -9,132 +9,35 @@ into a single directory named with a suffix of .otiod. """ -import os -import shutil - -from . import ( - file_bundle_utils as utils, - otio_json, -) - from .. import ( - exceptions, - url_utils, + _otio ) -import pathlib -import urllib.parse as urlparse - def read_from_file( filepath, # convert the media_reference paths to absolute paths absolute_media_reference_paths=False, ): - result = otio_json.read_from_file( - os.path.join(filepath, utils.BUNDLE_PLAYLIST_PATH) - ) - - if not absolute_media_reference_paths: - return result - - for cl in result.find_clips(): - try: - source_fpath = cl.media_reference.target_url - except AttributeError: - continue - - rel_path = urlparse.urlparse(source_fpath).path - new_fpath = url_utils.url_from_filepath( - os.path.join(filepath, rel_path) - ) - - cl.media_reference.target_url = new_fpath - - return result + options = _otio.bundle.ReadOptions() + options.absolute_media_reference_paths = absolute_media_reference_paths + return _otio.bundle.read_otiod(filepath, options) def write_to_file( input_otio, filepath, - # see documentation in file_bundle_utils for more information on the - # media_policy - media_policy=utils.MediaReferencePolicy.ErrorIfNotFile, + relative_media_base_dir=None, + # see documentation bundle.h for more information on the media_policy + media_policy=_otio.bundle.MediaReferencePolicy.error_if_not_file, dryrun=False ): + options = _otio.bundle.WriteOptions() + options.relative_media_base_dir = relative_media_base_dir + options.policy = media_policy - if os.path.exists(filepath): - raise exceptions.OTIOError( - f"'{filepath}' exists, will not overwrite." - ) - - if not os.path.exists(os.path.dirname(filepath)): - raise exceptions.OTIOError( - f"Directory '{os.path.dirname(filepath)}' does not exist, cannot" - f" create '{filepath}'." - ) - - if not os.path.isdir(os.path.dirname(filepath)): - raise exceptions.OTIOError( - f"'{os.path.dirname(filepath)}' is not a directory, cannot create" - f" '{filepath}'." - ) - - # general algorithm for the file bundle adapters: - # ------------------------------------------------------------------------- - # - build file manifest (list of paths to files on disk that will be put - # into the archive) - # - build a mapping of path to file on disk to url to put into the media - # reference in the result - # - relink the media references to point at the final location inside the - # archive - # - build the resulting structure (zip file, directory) - # ------------------------------------------------------------------------- - - result_otio, path_to_mr_map = utils._prepped_otio_for_bundle_and_manifest( - input_otio, - media_policy, - "OTIOD" - ) - - # dryrun reports the total size of files if dryrun: - return utils._total_file_size_of(path_to_mr_map.keys()) - - abspath_to_output_path_map = {} - - # relink all the media references to their target paths - for abspath, references in path_to_mr_map.items(): - target = os.path.join( - filepath, - utils.BUNDLE_DIR_NAME, - os.path.basename(abspath) - ) - - # conform to posix style paths inside the bundle, so that they are - # portable between windows and *nix style environments - final_path = str(pathlib.Path(target).as_posix()) - - # cache the output path - abspath_to_output_path_map[abspath] = final_path - - for mr in references: - # author the relative path from the root of the bundle in url - # form into the target_url - mr.target_url = url_utils.url_from_filepath( - os.path.relpath(final_path, filepath) - ) - - os.mkdir(filepath) - - otio_json.write_to_file( - result_otio, - os.path.join(filepath, utils.BUNDLE_PLAYLIST_PATH) - ) - - # write the media files - os.mkdir(os.path.join(filepath, utils.BUNDLE_DIR_NAME)) - for src, dst in abspath_to_output_path_map.items(): - shutil.copyfile(src, dst) + return _otio.bundle.dry_run(input_otio, options) + _otio.bundle.write_otiod(input_otio, filepath, options) return diff --git a/src/py-opentimelineio/opentimelineio/adapters/otioz.py b/src/py-opentimelineio/opentimelineio/adapters/otioz.py index 0fae0a2946..604bbe4f25 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otioz.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otioz.py @@ -16,142 +16,36 @@ read on unix and windows platforms. """ -import os -import zipfile - from .. import ( - exceptions, - url_utils, -) - -from . import ( - file_bundle_utils as utils, - otio_json, + _otio ) -import pathlib - def read_from_file( filepath, # if provided, will extract contents of zip to this directory extract_to_directory=None, ): - if not zipfile.is_zipfile(filepath): - raise exceptions.OTIOError(f"Not a zipfile: {filepath}") - - if extract_to_directory: - output_media_directory = os.path.join( - extract_to_directory, - utils.BUNDLE_DIR_NAME - ) - - if not os.path.exists(extract_to_directory): - raise exceptions.OTIOError( - f"Directory '{extract_to_directory}' does not exist, cannot" - f" unpack otioz there." - ) - - if os.path.exists(output_media_directory): - raise exceptions.OTIOError( - f"Error: '{output_media_directory}' already exists on disk, " - f"cannot overwrite while unpacking OTIOZ file '{filepath}'." - ) - - with zipfile.ZipFile(filepath, 'r') as zi: - result = otio_json.read_from_string( - zi.read(utils.BUNDLE_PLAYLIST_PATH) - ) - - if extract_to_directory: - zi.extractall(extract_to_directory) - - return result + options = _otio.bundle.ReadOptions() + if extract_to_directory is not None: + options.extract_path = extract_to_directory + return _otio.bundle.read_otioz(filepath, options) def write_to_file( input_otio, filepath, - # see documentation in file_bundle_utils for more information on the - # media_policy - media_policy=utils.MediaReferencePolicy.ErrorIfNotFile, + relative_media_base_dir=None, + # see documentation in bundle. for more information on the media_policy + media_policy=_otio.bundle.MediaReferencePolicy.error_if_not_file, dryrun=False ): - if os.path.exists(filepath): - raise exceptions.OTIOError( - f"'{filepath}' exists, will not overwrite." - ) - - # general algorithm for the file bundle adapters: - # ------------------------------------------------------------------------- - # - build file manifest (list of paths to files on disk that will be put - # into the archive) - # - build a mapping of path to file on disk to url to put into the media - # reference in the result - # - relink the media references to point at the final location inside the - # archive - # - build the resulting structure (zip file, directory) - # ------------------------------------------------------------------------- + options = _otio.bundle.WriteOptions() + options.relative_media_base_dir = relative_media_base_dir + options.policy = media_policy - result_otio, path_to_mr_map = utils._prepped_otio_for_bundle_and_manifest( - input_otio, - media_policy, - "OTIOZ" - ) - - # dryrun reports the total size of files if dryrun: - return utils._total_file_size_of(path_to_mr_map.keys()) - - abspath_to_output_path_map = {} - - # relink all the media references to their target paths - for abspath, references in path_to_mr_map.items(): - target = os.path.join(utils.BUNDLE_DIR_NAME, os.path.basename(abspath)) - - # conform to posix style paths inside the bundle, so that they are - # portable between windows and *nix style environments - final_path = str(pathlib.Path(target).as_posix()) - - # cache the output path - abspath_to_output_path_map[abspath] = final_path - - for mr in references: - # author the final_path in url form into the target_url - mr.target_url = url_utils.url_from_filepath(final_path) - - # write the otioz file to the temp directory - otio_str = otio_json.write_to_string(result_otio) - - with zipfile.ZipFile(filepath, mode='w') as target: - # write the version file (compressed) - target.writestr( - utils.BUNDLE_VERSION_FILE, - utils.BUNDLE_VERSION, - # XXX: OTIOZ was introduced when python 2.7 was still a supported - # platform. The newer algorithms, like BZIP2 and LZMA, are not - # available in python2, so it uses the zlib based - # ZIP_DEFLATED. Now that OTIO is Python3+, this could switch - # to using BZIP2 or LZMA instead... with the caveat that this - # would make OTIOZ files incompatible with python 2 based OTIO - # installs. - # - # For example, if we used ZIP_LZMA, then otio release v0.15 - # would still be able to open these files as long as the - # python interpreter was version 3+. - compress_type=zipfile.ZIP_DEFLATED - ) - - # write the OTIO (compressed) - target.writestr( - utils.BUNDLE_PLAYLIST_PATH, - otio_str, - # XXX: See comment above about ZIP_DEFLATED vs other algorithms - compress_type=zipfile.ZIP_DEFLATED - ) - - # write the media (uncompressed) - for src, dst in abspath_to_output_path_map.items(): - target.write(src, dst, compress_type=zipfile.ZIP_STORED) + return _otio.bundle.dry_run(input_otio, options) + _otio.bundle.write_otioz(input_otio, filepath, options) return diff --git a/src/py-opentimelineio/opentimelineio/url_utils.py b/src/py-opentimelineio/opentimelineio/url_utils.py deleted file mode 100644 index d99cc9ba90..0000000000 --- a/src/py-opentimelineio/opentimelineio/url_utils.py +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright Contributors to the OpenTimelineIO project - -"""Utilities for conversion between urls and file paths""" - -import os -import urllib -from urllib import request -import pathlib - - -def url_from_filepath(fpath): - """Convert a filesystem path to an url in a portable way. - - ensures that `fpath` conforms to the following pattern: - * if it is an absolute path, "file:///path/to/thing" - * if it is a relative path, "path/to/thing" - - In other words, if you pass in: - * "/var/tmp/thing.otio" -> "file:///var/tmp/thing.otio" - * "subdir/thing.otio" -> "tmp/thing.otio" - """ - - try: - # appears to handle absolute windows paths better, which are absolute - # and start with a drive letter. - return urllib.parse.unquote(pathlib.PurePath(fpath).as_uri()) - except ValueError: - # scheme is "file" for absolute paths, else "" - scheme = "file" if os.path.isabs(fpath) else "" - - # handles relative paths - return urllib.parse.urlunparse( - urllib.parse.ParseResult( - scheme=scheme, - path=fpath, - netloc="", - params="", - query="", - fragment="" - ) - ) - - -def filepath_from_url(urlstr): - """ - Take an url and return a filepath. - - URLs can either be encoded according to the `RFC 3986`_ standard or not. - Additionally, Windows mapped drive letter and UNC paths need to be - accounted for when processing URL(s); however, there are `ongoing - discussions`_ about how to best handle this within Python developer - community. This function is meant to cover these scenarios in the interim. - - .. _RFC 3986: https://tools.ietf.org/html/rfc3986#section-2.1 - .. _ongoing discussions: https://discuss.python.org/t/file-uris-in-python/15600 - """ - - # Parse provided URL - parsed_result = urllib.parse.urlparse(urlstr) - - # De-encode the parsed path - decoded_parsed_path = urllib.parse.unquote(parsed_result.path) - - # Convert the parsed URL to a path - filepath = pathlib.PurePath( - request.url2pathname(decoded_parsed_path) - ) - - # If the network location is a window drive, reassemble the path - if pathlib.PureWindowsPath(parsed_result.netloc).drive: - filepath = pathlib.PurePath(parsed_result.netloc + decoded_parsed_path) - - # If the specified index is a windows drive, then append it to the other - # parts - elif pathlib.PureWindowsPath(filepath.parts[0]).drive: - filepath = pathlib.PurePosixPath(filepath.drive, *filepath.parts[1:]) - - # If the specified index is a windows drive, then offset the path - elif ( - # relative paths may not have a parts[1] - len(filepath.parts) > 1 - and pathlib.PureWindowsPath(filepath.parts[1]).drive - ): - # Remove leading "/" if/when `request.url2pathname` yields - # "/S:/path/file.ext" - filepath = pathlib.PurePosixPath(*filepath.parts[1:]) - - # Should catch UNC paths, - # as parsing "file:///some/path/to/file.ext" doesn't provide a netloc - elif parsed_result.netloc and parsed_result.netloc != 'localhost': - # Paths of type: "file://host/share/path/to/file.ext" provide "host" as - # netloc - filepath = pathlib.PurePath( - '//', - parsed_result.netloc + decoded_parsed_path - ) - - # Executing `as_posix` on Windows seems to generate a path with only 1 - # leading `/`, so we insert another `/` at the front of the string path - # to match Linux and Windows UNC conventions and return it. - conformed_filepath = filepath.as_posix() - if not conformed_filepath.startswith('//'): - conformed_filepath = '/' + conformed_filepath - return conformed_filepath - - # Convert "\" to "/" if needed - return filepath.as_posix() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index afef11a474..43777d5924 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,7 +17,17 @@ foreach(test ${tests_opentime}) WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endforeach() -list(APPEND tests_opentimelineio test_clip test_serialization test_serializableCollection test_stack_algo test_timeline test_track test_editAlgorithm test_composition) +list(APPEND + tests_opentimelineio + test_bundle + test_clip + test_composition + test_editAlgorithm + test_serialization + test_serializableCollection + test_stack_algo + test_timeline + test_track) foreach(test ${tests_opentimelineio}) add_executable(${test} utils.h utils.cpp ${test}.cpp) diff --git a/tests/test_bundle.cpp b/tests/test_bundle.cpp new file mode 100644 index 0000000000..5d8667613b --- /dev/null +++ b/tests/test_bundle.cpp @@ -0,0 +1,575 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project + +#include "utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace OTIO_NS; +using namespace OTIO_NS::bundle; + +// Utility for creating a simple timeline +SerializableObject::Retainer create_simple_timeline() +{ + SerializableObject::Retainer tl(new Timeline); + + SerializableObject::Retainer tr = new Track( + "video", + TimeRange(0, 48, 24), + Track::Kind::video); + tl->tracks()->append_child(tr); + + SerializableObject::Retainer cl = new Clip( + "video clip 1", + nullptr, + TimeRange(0, 24, 24)); + tr->append_child(cl); + + cl = new Clip( + "video clip 2", + nullptr, + TimeRange(0, 24, 24)); + tr->append_child(cl); + + tr = new Track( + "audio", + TimeRange(0, 48, 24), + Track::Kind::audio); + tl->tracks()->append_child(tr); + + cl = new Clip( + "audio clip 1", + nullptr, + TimeRange(0, 24, 48)); + tr->append_child(cl); + + return tl; +} + +// Utility to find a clip by name +Clip* find_clip_by_name(Timeline* timeline, std::string const& name) +{ + for (auto clip : timeline->find_clips()) + { + if (name == clip->name()) + { + return clip; + } + } + return nullptr; +} + +// Utility for creating an empty file +void create_file(std::filesystem::path const& path) +{ + std::filesystem::create_directories(path.parent_path()); + std::ofstream fs(path); +} + +// Utility for creating empty files for each media reference +void create_refs(Timeline* timeline, std::filesystem::path const& path) +{ + for (auto const& clip : timeline->find_clips()) + { + for (auto const& ref : clip->media_references()) + { + if (auto ext = dynamic_cast(ref.second)) + { + if (auto const file = file_from_url(ext->target_url())) + { + auto const file_path = std::filesystem::u8path(*file); + create_file(file_path.is_relative() ? + path / file_path : + file_path); + } + } + else if (auto seq = dynamic_cast(ref.second)) + { + auto const base = file_from_url(seq->target_url_base()); + for (int frame = seq->start_frame(); + frame <= seq->end_frame(); + frame += seq->frame_step()) + { + if (auto const file = file_from_url(seq->target_url_for_image_number(frame))) + { + auto const file_path = std::filesystem::u8path(*file); + create_file(file_path.is_relative() ? + path / file_path : + file_path); + } + } + } + } + } +} + +// Utility to compare media reference filenames between timeline +void compare_filenames(Timeline* a, Timeline* b) +{ + const auto a_clips = a->find_clips(); + const auto b_clips = b->find_clips(); + assertEqual(a_clips.size(), b_clips.size()); + for (size_t i = 0; i < a_clips.size(); ++i) + { + auto a_refs = a_clips[i]->media_references(); + auto b_refs = b_clips[i]->media_references(); + assertEqual(a_refs.size(), b_refs.size()); + for (auto a = a_refs.begin(), b = b_refs.begin(); + a != a_refs.end() && b != b_refs.end(); + ++a, ++b) + { + auto a_ext = dynamic_cast(a->second); + auto b_ext = dynamic_cast(b->second); + if (a_ext && b_ext) + { + auto const a_file = file_from_url(a_ext->target_url()); + auto const b_file = file_from_url(b_ext->target_url()); + if (a_file && b_file) + { + assertEqual( + std::filesystem::u8path(*a_file).filename(), + std::filesystem::u8path(*b_file).filename()); + } + } + auto a_seq = dynamic_cast(a->second); + auto b_seq = dynamic_cast(b->second); + if (a_seq && b_seq) + { + assertEqual(a_seq->name_prefix(), b_seq->name_prefix()); + assertEqual(a_seq->name_suffix(), b_seq->name_suffix()); + } + } + } +} + +int +main(int argc, char** argv) +{ + Tests tests; + + tests.add_test("test_file_from_url", [] { + std::map const urls = + { + { + // windows encoded url + "file://host/S%3a/path/file.ext", + "S:/path/file.ext" + }, + { + // windows drive url + "file://S:/path/file.ext", + "S:/path/file.ext" + }, + { + // windows encoded_unc url + "file://unc/path/sub%20dir/file.ext", + "//unc/path/sub dir/file.ext" + }, + { + // windows unc url + "file://unc/path/sub dir/file.ext", + "//unc/path/sub dir/file.ext" + }, + { + // posix localhost url + "file://localhost/path/sub dir/file.ext", + "/path/sub dir/file.ext" + }, + { + // posix encoded url + "file:///path/sub%20dir/file.ext", + "/path/sub dir/file.ext" + }, + { + // posix url + "file:///path/sub dir/file.ext", + "/path/sub dir/file.ext" + }, + }; + for (auto i : urls) + { + std::string const file = file_from_url(i.first).value(); + assertEqual(file, i.second); + } + }); + + tests.add_test("test_otioz_round_trip", [] { + TempDir temp; + + // Create a timeline and media references + auto tl = create_simple_timeline(); + find_clip_by_name(tl, "video clip 1")->set_media_reference( + new ExternalReference("video1.mov")); + find_clip_by_name(tl, "video clip 2")->set_media_reference( + new ImageSequenceReference( + "", + "render.", + ".exr", + 0, 1, 24, 0, + ImageSequenceReference::MissingFramePolicy::error, + TimeRange(0, 24, 24))); + find_clip_by_name(tl, "audio clip 1")->set_media_references( + { + { "wav", new ExternalReference("audio.wav") }, + { "absolute_path", new ExternalReference((temp.path() / "audio.mp3").u8string()) }, + { "sub_dir", new ExternalReference("sub_dir/audio.ogg") } + }, + "wav"); + create_refs(tl, temp.path()); + + // Dry run + std::string const otioz_path = + (temp.path() / "round_trip.otioz").u8string(); + WriteOptions write_options; + write_options.relative_media_base_dir = temp.path().u8string(); + OTIO_NS::ErrorStatus error; + auto const size = dry_run(tl, write_options, &error); + assertTrue(size.has_value()); + assertTrue(*size > 0); + + // Write the otioz + assertTrue(write_otioz(tl, otioz_path, write_options, &error)); + + // Read the otioz and compare with the original + auto result = dynamic_cast(read_otioz( + otioz_path, + ReadOptions(), + &error)); + assertNotNull(result); + compare_filenames(tl, result); + + // Read the otioz and extract the contents + ReadOptions read_options; + auto const extract_path = temp.path() / "extract"; + read_options.extract_path = extract_path.u8string(); + result = dynamic_cast(read_otioz( + otioz_path, + read_options, + &error)); + assertNotNull(result); + auto const media_path = extract_path / media_dir; + assertTrue(std::filesystem::exists(media_path / "video1.mov")); + for (int i = 0; i < 24; ++i) + { + std::stringstream ss; + ss << "render." << i << ".exr"; + assertTrue(std::filesystem::exists(media_path / ss.str())); + } + assertTrue(std::filesystem::exists(media_path / "audio.wav")); + assertTrue(std::filesystem::exists(media_path / "audio.mp3")); + assertTrue(std::filesystem::exists(media_path / "audio.ogg")); + }); + + tests.add_test("test_otiod_round_trip", [] { + TempDir temp; + + // Create a timeline and media references + auto tl = create_simple_timeline(); + find_clip_by_name(tl, "video clip 1")->set_media_reference( + new ExternalReference("video1.mov")); + find_clip_by_name(tl, "video clip 2")->set_media_reference( + new ImageSequenceReference( + "", + "render.", + ".exr", + 0, 1, 24, 0, + ImageSequenceReference::MissingFramePolicy::error, + TimeRange(0, 24, 24))); + find_clip_by_name(tl, "audio clip 1")->set_media_references( + { + { "wav", new ExternalReference("audio.wav") }, + { "absolute_path", new ExternalReference((temp.path() / "audio.mp3").u8string()) }, + { "sub_dir", new ExternalReference("sub_dir/audio.ogg") } + }, + "wav"); + create_refs(tl, temp.path()); + + // Write the otiod + std::string const otiod_path = + (temp.path() / "round_trip.otiod").u8string(); + WriteOptions write_options; + write_options.relative_media_base_dir = temp.path().u8string(); + OTIO_NS::ErrorStatus error; + assertTrue(write_otiod(tl, otiod_path, write_options, &error)); + + // Read the otiod and compare with the original + ReadOptions read_options; + read_options.absolute_media_reference_paths = true; + auto result = dynamic_cast(read_otiod( + otiod_path, + read_options, + &error)); + assertNotNull(result); + compare_filenames(tl, result); + + // Check that the paths are absolute + auto cl = find_clip_by_name(result, "video clip 1"); + auto file = file_from_url(dynamic_cast( + cl->media_reference())->target_url()); + assertTrue(file.has_value()); + assertTrue(std::filesystem::u8path(*file).is_absolute()); + cl = find_clip_by_name(result, "video clip 2"); + file = file_from_url(dynamic_cast( + cl->media_reference())->target_url_for_image_number(0)); + assertTrue(file.has_value()); + assertTrue(std::filesystem::u8path(*file).is_absolute()); + }); + + tests.add_test("test_otioz_media_policy", [] { + + // Create a timeline with file and non-file references + auto tl = create_simple_timeline(); + find_clip_by_name(tl, "video clip 1")->set_media_reference( + new ExternalReference("video1.mov")); + find_clip_by_name(tl, "video clip 2")->set_media_reference( + new GeneratorReference( + "gradient", + "gradient", + TimeRange(0, 24, 24), + {}, + { + { "meta", "data" } + })); + + // error_if_not_file + { + TempDir temp; + create_refs(tl, temp.path()); + + std::string const otioz_path = + (temp.path() / "error_if_not_file.otioz").u8string(); + WriteOptions write_options; + write_options.relative_media_base_dir = temp.path().u8string(); + write_options.policy = MediaReferencePolicy::error_if_not_file; + OTIO_NS::ErrorStatus error; + assertFalse(dry_run(tl, write_options, &error).has_value()); + assertFalse(write_otioz(tl, otioz_path, write_options, &error)); + } + + // missing_if_not_file + { + TempDir temp; + create_refs(tl, temp.path()); + + std::string const otioz_path = + (temp.path() / "missing_if_not_file.otioz").u8string(); + WriteOptions write_options; + write_options.relative_media_base_dir = temp.path().u8string(); + write_options.policy = MediaReferencePolicy::missing_if_not_file; + OTIO_NS::ErrorStatus error; + assertTrue(dry_run(tl, write_options, &error).has_value()); + assertTrue(write_otioz(tl, otioz_path, write_options, &error)); + + auto result = dynamic_cast(read_otioz( + otioz_path, + ReadOptions(), + &error)); + assertNotNull(result); + assertNotNull(dynamic_cast( + find_clip_by_name(result, "video clip 2")->media_reference())); + } + + // all_missing + { + TempDir temp; + create_refs(tl, temp.path()); + + std::string const otioz_path = + (temp.path() / "all_missing.otioz").u8string(); + WriteOptions write_options; + write_options.relative_media_base_dir = temp.path().u8string(); + write_options.policy = MediaReferencePolicy::all_missing; + OTIO_NS::ErrorStatus error; + assertTrue(dry_run(tl, write_options, &error).has_value()); + assertTrue(write_otioz(tl, otioz_path, write_options, &error)); + + auto result = dynamic_cast(read_otioz( + otioz_path, + ReadOptions(), + &error)); + assertNotNull(result); + assertNotNull(dynamic_cast( + find_clip_by_name(result, "video clip 1")->media_reference())); + assertNotNull(dynamic_cast( + find_clip_by_name(result, "video clip 2")->media_reference())); + } + }); + + tests.add_test("test_otioz_empty", [] { + TempDir temp; + SerializableObject::Retainer tl(new Timeline); + + auto const otioz_path = (temp.path() / "empty.otioz").u8string(); + OTIO_NS::ErrorStatus error; + assertTrue(write_otioz(tl, otioz_path, WriteOptions(), &error)); + + auto result = dynamic_cast(read_otioz( + otioz_path, ReadOptions(), &error)); + assertNotNull(result); + assertEqual(result->find_clips().size(), 0); + }); + + tests.add_test("test_otiod_empty", [] { + TempDir temp; + SerializableObject::Retainer tl(new Timeline); + + auto const otiod_path = (temp.path() / "empty.otiod").u8string(); + OTIO_NS::ErrorStatus error; + assertTrue(write_otiod(tl, otiod_path, WriteOptions(), &error)); + + auto result = dynamic_cast(read_otiod( + otiod_path, ReadOptions(), &error)); + assertNotNull(result); + assertEqual(result->find_clips().size(), 0); + }); + + tests.add_test("test_otioz_error", [] { + TempDir temp; + + // Create a timeline + auto tl = create_simple_timeline(); + + // Write the otioz + std::string otioz_path = + (temp.path() / "error.otioz").u8string(); + OTIO_NS::ErrorStatus error; + assertTrue(write_otioz(tl, otioz_path, WriteOptions(), &error)); + + // Write the otioz (error on overwrite) + assertFalse(write_otioz(tl, otioz_path, WriteOptions(), &error)); + std::filesystem::remove(otioz_path); + + // Write the otioz (error on missing media) + find_clip_by_name(tl, "video clip 1")->set_media_reference( + new ExternalReference("video.mov")); + assertFalse(write_otioz(tl, otioz_path, WriteOptions(), &error)); + std::filesystem::remove(otioz_path); + + // Write the otioz (error on overwrite media) + find_clip_by_name(tl, "video clip 2")->set_media_reference( + new ExternalReference("sub_dir/video.mov")); + create_refs(tl, temp.path()); + assertFalse(write_otioz(tl, otioz_path, WriteOptions(), &error)); + std::filesystem::remove(otioz_path); + + // Write the otioz (error on input path) + otioz_path = (temp.path() / "subdir" / "error.otioz").u8string(); + assertFalse(write_otioz(tl, otioz_path, WriteOptions(), &error)); + std::filesystem::remove(otioz_path); + }); + + tests.add_test("test_otiod_error", [] { + TempDir temp; + + // Create a timeline + auto tl = create_simple_timeline(); + + // Write the otiod + std::string otiod_path = + (temp.path() / "error.otiod").u8string(); + OTIO_NS::ErrorStatus error; + assertTrue(write_otiod(tl, otiod_path, WriteOptions(), &error)); + + // Write the otiod (error on overwrite) + assertFalse(write_otiod(tl, otiod_path, WriteOptions(), &error)); + std::filesystem::remove_all(otiod_path); + + // Write the otiod (missing media) + find_clip_by_name(tl, "video clip 1")->set_media_reference( + new ExternalReference("video.mov")); + assertFalse(write_otiod(tl, otiod_path, WriteOptions(), &error)); + std::filesystem::remove_all(otiod_path); + + // Write the otiod (error on overwrite media) + find_clip_by_name(tl, "video clip 2")->set_media_reference( + new ExternalReference("sub_dir/video.mov")); + create_refs(tl, temp.path()); + assertFalse(write_otiod(tl, otiod_path, WriteOptions(), &error)); + std::filesystem::remove_all(otiod_path); + + // Write the otiod (error on input path) + otiod_path = (temp.path() / "subdir" / "error.otiod").u8string(); + assertFalse(write_otiod(tl, otiod_path, WriteOptions(), &error)); + std::filesystem::remove_all(otiod_path); + }); + + tests.add_test("test_otioz_zip64", [] { + TempDir temp; + + // Create a timeline and media references + // + // To test 64-bit ZIP functionality: + // * Resize multiple media files > 4GB + // * Set the number of sequence images > max 16-bit value + // * Note that these tests need ~24GB disk space + auto tl = create_simple_timeline(); + std::string const large_file = "video1.mov"; + find_clip_by_name(tl, "video clip 1")->set_media_reference( + new ExternalReference(large_file)); + find_clip_by_name(tl, "video clip 2")->set_media_reference( + new ImageSequenceReference( + "", + "render.", + ".exr", + 0, 1, 24, 0, + ImageSequenceReference::MissingFramePolicy::error, + TimeRange(0, std::numeric_limits::max() + 1, 24))); + std::string const large_file_2 = "audio.wav"; + find_clip_by_name(tl, "audio clip 1")->set_media_reference( + new ExternalReference(large_file_2)); + create_refs(tl, temp.path()); + + // Resize media files > 4GB + const std::uintmax_t gigabyte = 1024 * 1024 * 1024; + std::uintmax_t const large_file_size = 4 * gigabyte; + std::filesystem::resize_file(temp.path() / large_file, large_file_size); + std::filesystem::resize_file(temp.path() / large_file_2, large_file_size); + + // Dry run + std::string const otioz_path = (temp.path() / "zip64.otioz").u8string(); + WriteOptions write_options; + write_options.relative_media_base_dir = temp.path().u8string(); + OTIO_NS::ErrorStatus error; + auto const dry_run_size = dry_run(tl, write_options, &error); + assertTrue(dry_run_size.has_value()); + assertTrue(*dry_run_size > large_file_size * 2); + + // Write the otioz + assertTrue(write_otioz(tl, otioz_path, write_options, &error)); + assertTrue(std::filesystem::file_size(otioz_path) >= *dry_run_size); + + // Read the otioz and extract the contents + ReadOptions read_options; + auto const extract_path = temp.path() / "extract"; + read_options.extract_path = extract_path.u8string(); + auto result = dynamic_cast(read_otioz( + otioz_path, + read_options, + &error)); + assertNotNull(result); + assertEqual( + std::filesystem::file_size(extract_path / "media" / large_file), + large_file_size); + assertEqual( + std::filesystem::file_size(extract_path / "media" / large_file_2), + large_file_size); + }); + + // \todo Add a test case for "zip slip", where ZIP files have malicious + // entries. This requires manually creating a ZIP file with entries + // outside of the ZIP directory (eg., "../../../passwd"). The read_otioz + // should catch these and cause an error. + + tests.run(argc, argv); + return 0; +} diff --git a/tests/test_otiod.py b/tests/test_otiod.py index aef9eef8e0..b1a33b55ed 100644 --- a/tests/test_otiod.py +++ b/tests/test_otiod.py @@ -7,185 +7,39 @@ import unittest import os +import pathlib import tempfile import opentimelineio as otio -from opentimelineio import test_utils as otio_test_utils -from opentimelineio.adapters import file_bundle_utils - -SAMPLE_DATA_DIR = os.path.join(os.path.dirname(__file__), "sample_data") -SCREENING_EXAMPLE_PATH = os.path.join(SAMPLE_DATA_DIR, "screening_example.otio") - -MEDIA_EXAMPLE_PATH_REL = os.path.relpath( - os.path.join( - SAMPLE_DATA_DIR, - "OpenTimelineIO@3xDark.png" - ) -) -MEDIA_EXAMPLE_PATH_URL_REL = otio.url_utils.url_from_filepath( - MEDIA_EXAMPLE_PATH_REL -) -MEDIA_EXAMPLE_PATH_ABS = os.path.abspath( - MEDIA_EXAMPLE_PATH_REL.replace( - "3xDark", - "3xLight" - ) -) -MEDIA_EXAMPLE_PATH_URL_ABS = otio.url_utils.url_from_filepath( - MEDIA_EXAMPLE_PATH_ABS -) +import opentimelineio.test_utils as otio_test_utils class OTIODTester(unittest.TestCase, otio_test_utils.OTIOAssertions): - def setUp(self): - tl = otio.adapters.read_from_file(SCREENING_EXAMPLE_PATH) - - # convert to contrived local reference - last_rel = False - for cl in tl.find_clips(): - # vary the relative and absolute paths, make sure that both work - next_rel = ( - MEDIA_EXAMPLE_PATH_URL_REL if last_rel else MEDIA_EXAMPLE_PATH_URL_ABS - ) - last_rel = not last_rel - cl.media_reference = otio.schema.ExternalReference( - target_url=next_rel - ) - - self.tl = tl - - def test_file_bundle_manifest_missing_reference(self): - # all missing should be empty - result_otio, manifest = ( - file_bundle_utils._prepped_otio_for_bundle_and_manifest( - input_otio=self.tl, - media_policy=file_bundle_utils.MediaReferencePolicy.AllMissing, - adapter_name="TEST_NAME", - ) - ) - - self.assertEqual(manifest, {}) - for cl in result_otio.find_clips(): - self.assertIsInstance( - cl.media_reference, - otio.schema.MissingReference, - "{} is of type {}, not an instance of {}.".format( - cl.media_reference, - type(cl.media_reference), - type(otio.schema.MissingReference) - ) - ) - - def test_file_bundle_manifest(self): - result_otio, manifest = ( - file_bundle_utils._prepped_otio_for_bundle_and_manifest( - input_otio=self.tl, - media_policy=( - file_bundle_utils.MediaReferencePolicy.ErrorIfNotFile - ), - adapter_name="TEST_NAME", - ) - ) - - self.assertEqual(len(manifest.keys()), 2) - - files_in_manifest = set(manifest.keys()) - known_files = { - MEDIA_EXAMPLE_PATH_ABS: 5, - os.path.abspath(MEDIA_EXAMPLE_PATH_REL): 4 - } - - # should only contain absolute paths - self.assertEqual(files_in_manifest, set(known_files.keys())) - - for fname, count in known_files.items(): - self.assertEqual(len(manifest[fname]), count) - def test_round_trip(self): - with tempfile.NamedTemporaryFile(suffix=".otiod") as bogusfile: - tmp_path = bogusfile.name - otio.adapters.write_to_file(self.tl, tmp_path) - self.assertTrue(os.path.exists(tmp_path)) - - # by default will provide relative paths - result = otio.adapters.read_from_file( - tmp_path, - ) - - for cl in result.find_clips(): - self.assertNotEqual( - cl.media_reference.target_url, - MEDIA_EXAMPLE_PATH_URL_REL - ) - - # conform media references in input to what they should be in the output - for cl in self.tl.find_clips(): - # construct an absolute file path to the result - cl.media_reference.target_url = ( - otio.url_utils.url_from_filepath( - os.path.join( - otio.adapters.file_bundle_utils.BUNDLE_DIR_NAME, - os.path.basename(cl.media_reference.target_url) - ) - ) - ) - - self.assertJsonEqual(result, self.tl) - - def test_round_trip_all_missing_references(self): - with tempfile.NamedTemporaryFile(suffix=".otiod") as bogusfile: - tmp_path = bogusfile.name - otio.adapters.write_to_file( - self.tl, - tmp_path, - media_policy=( - otio.adapters.file_bundle_utils.MediaReferencePolicy.AllMissing - ) - ) - - # ...but can be optionally told to generate absolute paths - result = otio.adapters.read_from_file( - tmp_path, - absolute_media_reference_paths=True - ) - - for cl in result.find_clips(): - self.assertIsInstance( - cl.media_reference, - otio.schema.MissingReference - ) - - def test_round_trip_absolute_paths(self): - with tempfile.NamedTemporaryFile(suffix=".otiod") as bogusfile: - tmp_path = bogusfile.name - otio.adapters.write_to_file(self.tl, tmp_path) - - # ...but can be optionally told to generate absolute paths - result = otio.adapters.read_from_file( - tmp_path, - absolute_media_reference_paths=True - ) - - for cl in result.find_clips(): - self.assertNotEqual( - cl.media_reference.target_url, - MEDIA_EXAMPLE_PATH_URL_REL - ) - - # conform media references in input to what they should be in the output - for cl in self.tl.find_clips(): - # should be only field that changed - cl.media_reference.target_url = ( - otio.url_utils.url_from_filepath( - os.path.join( - tmp_path, - otio.adapters.file_bundle_utils.BUNDLE_DIR_NAME, - os.path.basename(cl.media_reference.target_url) - ) - ) - ) - - self.assertJsonEqual(result, self.tl) + with tempfile.TemporaryDirectory() as temp_dir: + + # Create a timeline + tl = otio.schema.Timeline() + tr = otio.schema.Track() + tl.tracks.append(tr) + cl = otio.schema.Clip() + tr.append(cl) + + # Add a media reference + ref = otio.schema.ExternalReference("video.mov") + cl.media_reference = ref + pathlib.Path(os.path.join(temp_dir, ref.target_url)).touch() + + # Write to otiod + otiod_path = os.path.join(temp_dir, "round_trip.otiod") + otio.adapters.write_to_file( + tl, + otiod_path, + relative_media_base_dir=temp_dir) + + # Read from otiod + result = otio.adapters.read_from_file(otiod_path) + self.assertIsNotNone(result) if __name__ == "__main__": diff --git a/tests/test_otioz.py b/tests/test_otioz.py index f9337b8e9a..9150a38fc8 100644 --- a/tests/test_otioz.py +++ b/tests/test_otioz.py @@ -7,244 +7,39 @@ import unittest import os +import pathlib import tempfile -import shutil - -import urllib.parse as urlparse import opentimelineio as otio import opentimelineio.test_utils as otio_test_utils -SAMPLE_DATA_DIR = os.path.join(os.path.dirname(__file__), "sample_data") -SCREENING_EXAMPLE_PATH = os.path.join(SAMPLE_DATA_DIR, "screening_example.otio") -MEDIA_EXAMPLE_PATH_REL = os.path.relpath( - os.path.join( - SAMPLE_DATA_DIR, - "OpenTimelineIO@3xDark.png" - ) -) -MEDIA_EXAMPLE_PATH_URL_REL = otio.url_utils.url_from_filepath( - MEDIA_EXAMPLE_PATH_REL -) -MEDIA_EXAMPLE_PATH_ABS = os.path.abspath( - MEDIA_EXAMPLE_PATH_REL.replace( - "3xDark", - "3xLight" - ) -) -MEDIA_EXAMPLE_PATH_URL_ABS = otio.url_utils.url_from_filepath( - MEDIA_EXAMPLE_PATH_ABS -) - class OTIOZTester(unittest.TestCase, otio_test_utils.OTIOAssertions): - def setUp(self): - tl = otio.adapters.read_from_file(SCREENING_EXAMPLE_PATH) - - # convert to contrived local reference - last_rel = False - for cl in tl.find_clips(): - # vary the relative and absolute paths, make sure that both work - next_rel = ( - MEDIA_EXAMPLE_PATH_URL_REL - if last_rel else MEDIA_EXAMPLE_PATH_URL_ABS - ) - last_rel = not last_rel - cl.media_reference = otio.schema.ExternalReference( - target_url=next_rel - ) - - self.tl = tl - - def test_dryrun(self): - # generate a fake name - with tempfile.NamedTemporaryFile(suffix=".otioz") as bogusfile: - fname = bogusfile.name - - # dryrun should compute what the total size of the zipfile will be. - size = otio.adapters.write_to_file(self.tl, fname, dryrun=True) - self.assertEqual( - size, - os.path.getsize(MEDIA_EXAMPLE_PATH_ABS) + - os.path.getsize(MEDIA_EXAMPLE_PATH_REL) - ) - - def test_not_a_file_error(self): - # dryrun should compute what the total size of the zipfile will be. - tmp_path = tempfile.mkstemp(suffix=".otioz", text=False)[1] - with tempfile.NamedTemporaryFile() as bogusfile: - fname = bogusfile.name - for cl in self.tl.find_clips(): - # write with a non-file schema - cl.media_reference = otio.schema.ExternalReference( - target_url=f"http://{fname}" - ) - with self.assertRaises(otio.exceptions.OTIOError): - otio.adapters.write_to_file(self.tl, tmp_path, dryrun=True) - - for cl in self.tl.find_clips(): - cl.media_reference = otio.schema.ExternalReference( - target_url=otio.url_utils.url_from_filepath(fname) - ) - with self.assertRaises(otio.exceptions.OTIOError): - otio.adapters.write_to_file(self.tl, tmp_path, dryrun=True) - - tempdir = tempfile.mkdtemp() - fname = tempdir - shutil.rmtree(tempdir) - for cl in self.tl.find_clips(): - cl.media_reference = otio.schema.ExternalReference(target_url=fname) - - def test_colliding_basename(self): - tempdir = tempfile.mkdtemp() - new_path = os.path.join( - tempdir, - os.path.basename(MEDIA_EXAMPLE_PATH_ABS) - ) - shutil.copyfile( - MEDIA_EXAMPLE_PATH_ABS, - new_path - ) - list(self.tl.find_clips())[0].media_reference.target_url = ( - otio.url_utils.url_from_filepath(new_path) - ) - - tmp_path = tempfile.mkstemp(suffix=".otioz", text=False)[1] - with self.assertRaises(otio.exceptions.OTIOError): - otio.adapters.write_to_file(self.tl, tmp_path) - - with self.assertRaises(otio.exceptions.OTIOError): - otio.adapters.write_to_file(self.tl, tmp_path, dryrun=True) - - shutil.rmtree(tempdir) - def test_round_trip(self): - with tempfile.NamedTemporaryFile(suffix=".otioz") as bogusfile: - tmp_path = bogusfile.name - otio.adapters.write_to_file(self.tl, tmp_path) - self.assertTrue(os.path.exists(tmp_path)) - - result = otio.adapters.read_from_file(tmp_path) - - for cl in result.find_clips(): - self.assertNotIn( - cl.media_reference.target_url, - [MEDIA_EXAMPLE_PATH_URL_ABS, MEDIA_EXAMPLE_PATH_URL_REL] - ) - # ensure that unix style paths are used, so that bundles created on - # windows are compatible with ones created on unix - self.assertFalse( - urlparse.urlparse( - cl.media_reference.target_url - ).path.startswith( - "media\\" - ) - ) - - # conform media references in input to what they should be in the output - for cl in self.tl.find_clips(): - # should be only field that changed - cl.media_reference.target_url = "media/{}".format( - os.path.basename(cl.media_reference.target_url) - ) - - self.assertJsonEqual(result, self.tl) - - def test_round_trip_with_extraction(self): - with tempfile.NamedTemporaryFile(suffix=".otioz") as bogusfile: - tmp_path = bogusfile.name - otio.adapters.write_to_file(self.tl, tmp_path) - self.assertTrue(os.path.exists(tmp_path)) - - tempdir = tempfile.mkdtemp() - result = otio.adapters.read_from_file( - tmp_path, - extract_to_directory=tempdir - ) - - # make sure that all the references are ExternalReference - for cl in result.find_clips(): - self.assertIsInstance( - cl.media_reference, - otio.schema.ExternalReference - ) - - # conform media references in input to what they should be in the output - for cl in self.tl.find_clips(): - # should be only field that changed - cl.media_reference.target_url = "media/{}".format( - os.path.basename(cl.media_reference.target_url) - ) - - self.assertJsonEqual(result, self.tl) - - # content file - self.assertTrue( - os.path.exists( - os.path.join( - tempdir, - otio.adapters.file_bundle_utils.BUNDLE_PLAYLIST_PATH - ) - ) - ) - - # media directory overall - self.assertTrue( - os.path.exists( - os.path.join( - tempdir, - otio.adapters.file_bundle_utils.BUNDLE_DIR_NAME - ) - ) - ) - - # actual media file - self.assertTrue( - os.path.exists( - os.path.join( - tempdir, - otio.adapters.file_bundle_utils.BUNDLE_DIR_NAME, - os.path.basename(MEDIA_EXAMPLE_PATH_URL_REL) - ) - ) - ) - - def test_round_trip_with_extraction_no_media(self): - with tempfile.NamedTemporaryFile(suffix=".otioz") as bogusfile: - tmp_path = bogusfile.name - otio.adapters.write_to_file( - self.tl, - tmp_path, - media_policy=( - otio.adapters.file_bundle_utils.MediaReferencePolicy.AllMissing - ), - ) - - tempdir = tempfile.mkdtemp() - result = otio.adapters.read_from_file( - tmp_path, - extract_to_directory=tempdir, - ) - - version_file_path = os.path.join( - tempdir, - otio.adapters.file_bundle_utils.BUNDLE_VERSION_FILE - ) - self.assertTrue(os.path.exists(version_file_path)) - with open(version_file_path) as fi: - self.assertEqual( - fi.read(), - otio.adapters.file_bundle_utils.BUNDLE_VERSION - ) - - # conform media references in input to what they should be in the output - for cl in result.find_clips(): - # should be all MissingReferences - self.assertIsInstance( - cl.media_reference, - otio.schema.MissingReference - ) - self.assertIn("original_target_url", cl.media_reference.metadata) + with tempfile.TemporaryDirectory() as temp_dir: + + # Create a timeline + tl = otio.schema.Timeline() + tr = otio.schema.Track() + tl.tracks.append(tr) + cl = otio.schema.Clip() + tr.append(cl) + + # Add a media reference + ref = otio.schema.ExternalReference("video.mov") + cl.media_reference = ref + pathlib.Path(os.path.join(temp_dir, ref.target_url)).touch() + + # Write to otioz + otioz_path = os.path.join(temp_dir, "round_trip.otioz") + otio.adapters.write_to_file( + tl, + otioz_path, + relative_media_base_dir=temp_dir) + + # Read from otiod + result = otio.adapters.read_from_file(otioz_path) + self.assertIsNotNone(result) if __name__ == "__main__": diff --git a/tests/test_url_conversions.py b/tests/test_url_conversions.py deleted file mode 100644 index 3adff3c03e..0000000000 --- a/tests/test_url_conversions.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright Contributors to the OpenTimelineIO project - -""" Unit tests of functions that convert between file paths and urls. """ - -import unittest -import os - -import opentimelineio as otio - -SAMPLE_DATA_DIR = os.path.join(os.path.dirname(__file__), "sample_data") -SCREENING_EXAMPLE_PATH = os.path.join(SAMPLE_DATA_DIR, "screening_example.otio") -MEDIA_EXAMPLE_PATH_REL = os.path.relpath( - os.path.join( - os.path.dirname(__file__), - "..", # root - "docs", - "_static", - "OpenTimelineIO@3xDark.png" - ) -) -MEDIA_EXAMPLE_PATH_URL_REL = otio.url_utils.url_from_filepath( - MEDIA_EXAMPLE_PATH_REL -) -MEDIA_EXAMPLE_PATH_ABS = os.path.abspath( - MEDIA_EXAMPLE_PATH_REL.replace( - "3xDark", - "3xLight" - ) -) -MEDIA_EXAMPLE_PATH_URL_ABS = otio.url_utils.url_from_filepath( - MEDIA_EXAMPLE_PATH_ABS -) - -WINDOWS_ENCODED_URL = "file://host/S%3a/path/file.ext" -WINDOWS_DRIVE_URL = "file://S:/path/file.ext" -WINDOWS_DRIVE_PATH = "S:/path/file.ext" - -WINDOWS_ENCODED_UNC_URL = "file://unc/path/sub%20dir/file.ext" -WINDOWS_UNC_URL = "file://unc/path/sub dir/file.ext" -WINDOWS_UNC_PATH = "//unc/path/sub dir/file.ext" - -POSIX_LOCALHOST_URL = "file://localhost/path/sub dir/file.ext" -POSIX_ENCODED_URL = "file:///path/sub%20dir/file.ext" -POSIX_URL = "file:///path/sub dir/file.ext" -POSIX_PATH = "/path/sub dir/file.ext" - - -class TestConversions(unittest.TestCase): - def test_roundtrip_abs(self): - self.assertTrue(MEDIA_EXAMPLE_PATH_URL_ABS.startswith("file://")) - full_path = os.path.abspath( - otio.url_utils.filepath_from_url(MEDIA_EXAMPLE_PATH_URL_ABS) - ) - - # should have reconstructed it by this point - self.assertEqual(full_path, MEDIA_EXAMPLE_PATH_ABS) - - def test_roundtrip_rel(self): - self.assertFalse(MEDIA_EXAMPLE_PATH_URL_REL.startswith("file://")) - - result = otio.url_utils.filepath_from_url(MEDIA_EXAMPLE_PATH_URL_REL) - - # should have reconstructed it by this point - self.assertEqual(os.path.normpath(result), MEDIA_EXAMPLE_PATH_REL) - - def test_windows_urls(self): - for url in (WINDOWS_ENCODED_URL, WINDOWS_DRIVE_URL): - processed_url = otio.url_utils.filepath_from_url(url) - self.assertEqual(processed_url, WINDOWS_DRIVE_PATH) - - def test_windows_unc_urls(self): - for url in (WINDOWS_ENCODED_UNC_URL, WINDOWS_UNC_URL): - processed_url = otio.url_utils.filepath_from_url(url) - self.assertEqual(processed_url, WINDOWS_UNC_PATH) - - def test_posix_urls(self): - for url in (POSIX_ENCODED_URL, POSIX_URL, POSIX_LOCALHOST_URL): - processed_url = otio.url_utils.filepath_from_url(url) - self.assertEqual(processed_url, POSIX_PATH) - - def test_relative_url(self): - # see github issue #1817 - when a relative URL has only one name after - # the "." (ie ./blah but not ./blah/blah) - self.assertEqual( - otio.url_utils.filepath_from_url(os.path.join(".", "docs")), - "docs", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/utils.cpp b/tests/utils.cpp index e49998daa1..d092b0619f 100644 --- a/tests/utils.cpp +++ b/tests/utils.cpp @@ -3,8 +3,9 @@ #include "utils.h" +#include #include -#include +#include void assertTrue(bool value) @@ -18,6 +19,60 @@ assertFalse(bool value) assert(!value); } +TempDir::TempDir() +{ + // Generate a unique temporary directory name without platform-specific + // routines. A 64-bit random value rendered as hex is plenty of entropy + // to avoid collisions between concurrent tests; the generator is + // thread_local so concurrent tests draw from independent streams. + static thread_local std::mt19937_64 generator(std::random_device{}()); + + auto const temp_root = std::filesystem::temp_directory_path(); + + // Retry on the (extremely unlikely) chance the name already exists. + for (int attempt = 0; attempt < 16; ++attempt) + { + uint64_t const rand_num = generator(); + + // 16 hex digits max for a 64-bit value; size the buffer generously. + char buf[32]; + auto const result = + std::to_chars(buf, buf + sizeof(buf), rand_num, 16); + std::string const name = + "otio_tmp_" + std::string(buf, result.ptr); + + auto candidate = temp_root / name; + + // create_directory returns true only if it created the directory, + // false (without setting ec) if it already existed, and sets ec on + // a real error. This gives a race-free "did I create it" check. + std::error_code ec; + if (std::filesystem::create_directory(candidate, ec)) + { + _path = std::move(candidate); + return; + } + if (ec) + { + throw std::runtime_error( + "cannot create temp directory in '" + + temp_root.u8string() + "': " + ec.message()); + } + + // Name collided; loop and try another. + } + + throw std::runtime_error( + "cannot create a unique temp directory in '" + + temp_root.u8string() + "'"); +} + +TempDir::~TempDir() +{ + std::error_code ec; + std::filesystem::remove_all(_path, ec); +} + void Tests::add_test(std::string const& name, std::function const& test) { diff --git a/tests/utils.h b/tests/utils.h index 3dbf18c31b..3fcf5daaba 100644 --- a/tests/utils.h +++ b/tests/utils.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,22 @@ assertNotNull(const void* a) assert(a != nullptr); } +class TempDir +{ +public: + TempDir(); + + ~TempDir(); + + TempDir(TempDir const&) = delete; + TempDir& operator=(TempDir const&) = delete; + + std::filesystem::path const& path() const { return _path; } + +private: + std::filesystem::path _path; +}; + class Tests { public: