diff --git a/osprey_coordinator/.gitignore b/osprey_coordinator/.gitignore new file mode 100644 index 0000000..ae739ac --- /dev/null +++ b/osprey_coordinator/.gitignore @@ -0,0 +1 @@ +src/proto/gen/* diff --git a/osprey_coordinator/BUILD b/osprey_coordinator/BUILD new file mode 100644 index 0000000..8698bfb --- /dev/null +++ b/osprey_coordinator/BUILD @@ -0,0 +1 @@ +exports_files(["Cargo.toml"]) diff --git a/osprey_coordinator/Cargo.toml b/osprey_coordinator/Cargo.toml new file mode 100644 index 0000000..a9256e4 --- /dev/null +++ b/osprey_coordinator/Cargo.toml @@ -0,0 +1,91 @@ +[package] +name = "smite_coordinator" +version = "0.1.0" +edition = "2021" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0" +arc-swap = "1.2" +async-channel = "2.3.1" +async-stream = "0.2" +async-trait = "0.1.22" +backoff = { version = "0.4", features = ["tokio"] } +base64 = "0.22" +bytes = "1.0" +bytes-utils = "0.1.2" +cadence = "1.1.0" +clap = { version = "4.1.8", features = ["derive", "env"] } +convert_case = "0.6.0" +dynamic-pool = "0.2.2" +env_logger = "0.7" +etcd = { git = "https://github.com/discord/rust-etcd", rev = "e3fc8242de7303582feca89d90230e89e744d06e" } +etcd_config_derive = { path = "etcd_config_derive", package = "smite_coordinator_etcd_config_derive" } +futures = "0.3" +fxhash = "0.2" +goauth = "0.10.0" +h2 = "0.3" +hdrhistogram = "7.5" +heck = "0.3" +hostname = "0.3.1" +http = "0.2" +http-body = "0.4.4" +hyper = { version = "0.14", features = ["backports", "deprecated"] } +indexmap = "1.9" +lazy_static = "1.4.0" +log = "0.4" +md5 = "0.7.0" +metrics_derive = { path = "metrics_derive", package = "smite_coordinator_metrics_derive" } +msgpack_simple = "1.0.2" +num_cpus = "1.0" +opentelemetry = "0.23" +opentelemetry-http = "0.12" +paste = "1.0.5" + +# metrics & tracing +parking_lot = "0.12" +percent-encoding = "2.1" +pin-project = "1.1.2" +prost = "0.12" +prost-types = "0.12" +rand = "0.8.5" +reqwest = { version = "0.12.22", features = ["json"] } +rmp-serde = "1.1.1" +rmpv = "1.0.0" +sentry = "0.31.8" +serde = { version = "1.0", features = ["rc", "derive"] } +serde-transcode = "1.1.1" +serde_json = "1.0" +smite_data_services_common = { path = "../common" } +smpl_jwt = "0.6.0" +strum_macros = "0.24" +thiserror = "2.0.12" +time = "0.2" +tink-aead = "0.2.4" +tink-core = "0.2.4" +tokio = { version = "1.4", features = ["full"] } +tokio-stream = "0.1" +tokio-util = { version = "0.7", features = ["time"] } +tonic = { version = "0.11", features = ["tls", "tls-roots"] } +tonic-health = "0.11.0" +tonic-reflection = "0.11.0" + +tower = "0.4" +tracing = "0.1" +tracing-core = "0.1" +tracing-opentelemetry = "0.24" +tracing-subscriber = "0.2" +trust-dns-resolver = "0.21" +uuid = { version = "1.0", features = ["v4"] } +which = "4.4.0" + +[build-dependencies] +glob = "0.3" +prost-build = { version = "0.12" } +tonic-build = "0.11" + +[dev-dependencies] + +[[bin]] +name = "smite_coordinator" +path = "src/main.rs" diff --git a/osprey_coordinator/build.rs b/osprey_coordinator/build.rs new file mode 100644 index 0000000..2b7a2ee --- /dev/null +++ b/osprey_coordinator/build.rs @@ -0,0 +1,65 @@ +fn main() -> Result<(), Box> { + let proto_root = std::path::Path::new("../../proto"); + let includes = vec!["../../proto/"]; + + let proto_paths = [ + proto_root.join("proto/osprey/rpc/actions/v1/action.proto"), + proto_root.join("proto/osprey/rpc/smite_coordinator/sync_action/v1/service.proto"), + proto_root + .join("proto/osprey/rpc/smite_coordinator/bidirectional_stream/v1/service.proto"), + proto_root.join("proto/osprey/rpc/common/v1/*.proto"), + ] + .into_iter() + .map(|pattern| glob::glob(pattern.to_str().unwrap())) + .collect::, _>>()?; + + let source_proto_files = proto_paths + .into_iter() + .flat_map(|paths| { + paths.filter_map(|maybe_path| { + maybe_path + .ok() + .map(|path| path.to_string_lossy().into_owned()) + }) + }) + .collect::>(); + + let mut prost_config = prost_build::Config::default(); + + // This is needed to convert enum variant -> string in order to get the action name in rust + prost_config.type_attribute("Action.data", "#[derive(strum_macros::Display)]"); + + let out_dir = std::env::var("OUT_DIR")?; + tonic_build::configure() + .file_descriptor_set_path(std::path::Path::new(&out_dir).join("descriptor.bin")) + .compile_with_config(prost_config, &source_proto_files, &includes)?; + + // Compile Google Cloud protos for inlined gcloud module + tonic_build::configure().build_server(false).compile( + &[ + proto_root.join("proto/google/api/annotations.proto"), + proto_root.join("proto/google/api/client.proto"), + proto_root.join("proto/google/api/field_behavior.proto"), + proto_root.join("proto/google/api/http.proto"), + proto_root.join("proto/google/api/resource.proto"), + proto_root.join("proto/google/iam/v1/iam_policy.proto"), + proto_root.join("proto/google/iam/v1/options.proto"), + proto_root.join("proto/google/iam/v1/policy.proto"), + proto_root.join("proto/google/longrunning/operations.proto"), + proto_root.join("proto/google/rpc/status.proto"), + proto_root.join("proto/google/type/expr.proto"), + proto_root.join("proto/google/pubsub/v1/pubsub.proto"), + proto_root.join("proto/google/cloud/kms/v1/service.proto"), + proto_root.join("proto/google/cloud/kms/v1/resources.proto"), + proto_root.join("proto/google/crypto/tink/aes_gcm.proto"), + ], + &[proto_root], + )?; + + tonic_build::configure().compile( + &[proto_root.join("proto/etcd_watcherd/v1/etcd_watcherd.proto")], + &[proto_root], + )?; + + Ok(()) +} diff --git a/osprey_coordinator/dev_env.sh b/osprey_coordinator/dev_env.sh new file mode 100644 index 0000000..a3badad --- /dev/null +++ b/osprey_coordinator/dev_env.sh @@ -0,0 +1,3 @@ +export ETCD_PEERS=http://127.0.0.1:2379 +export PUBSUB_EMULATOR_HOST=127.0.0.1:8085 +export POD_IP=127.0.0.1 diff --git a/osprey_coordinator/etcd_config_derive/Cargo.toml b/osprey_coordinator/etcd_config_derive/Cargo.toml new file mode 100644 index 0000000..82a0860 --- /dev/null +++ b/osprey_coordinator/etcd_config_derive/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "smite_coordinator_etcd_config_derive" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +proc-macro = true + +[dependencies] +convert_case = "0.6" +darling = "0.20" +proc-macro2 = "1.0.26" +quote = "1.0" +syn = { version = "2", features = ["full"] } diff --git a/osprey_coordinator/etcd_config_derive/src/disconfigs.rs b/osprey_coordinator/etcd_config_derive/src/disconfigs.rs new file mode 100644 index 0000000..30bb797 --- /dev/null +++ b/osprey_coordinator/etcd_config_derive/src/disconfigs.rs @@ -0,0 +1,140 @@ +use darling::{util::SpannedValue, FromDeriveInput, FromField, FromMeta}; +use quote::quote; + +#[derive(FromDeriveInput, Debug)] +#[darling(forward_attrs(config), supports(struct_any))] +pub(crate) struct Disconfig { + ident: syn::Ident, + attrs: Vec, + data: darling::ast::Data<(), SpannedValue>, +} + +const CONFIG_KEY: &'static str = "config.b64"; + +impl Disconfig { + pub(crate) fn impl_config(&self) -> darling::Result { + // Get config name from #[config] attribute. This name is used to create the key root. + let struct_name = &self.ident; + if self.attrs.len() > 1 { + return Err( + darling::Error::custom("Only one #[config] attribute is supported.") + .with_span(&self.attrs[1]), + ); + } + + let config_name_meta = &self + .attrs + .first() + .ok_or(darling::Error::custom("#[config] name must be set."))? + .meta; + + let config_name = ConfigName::from_meta(config_name_meta)?.name.value(); + let key_root = format!("/discord_configs/{config_name}/v1/"); + // Inside Disconfig struct, there should be one and only one ArcSwap field. + let fields = self.fields(); + let mut arc_swap_field_count = 0; + let mut config_field = None; + + for field in fields.iter() { + let field_ident = field.ident.clone().expect("Field must have an identifier"); + let field_type = &field.ty; + if let syn::Type::Path(type_path) = field_type { + let segment = type_path.path.segments.first(); + if let Some(path_segment) = segment { + // Make sure the field type is an ArcSwap. + if path_segment.ident.to_string() != "ArcSwap" + && path_segment.ident.to_string() != "arc_swap::ArcSwap" + { + continue; + } + arc_swap_field_count += 1; + config_field = Some(field_ident); + } + } + } + if arc_swap_field_count != 1 { + return Err( + darling::Error::custom("Struct must have one and only one ArcSwap field.") + .with_span(&self.ident.span()), + ); + } + + let field_name = config_field.unwrap(); + let config_field_name = field_name.to_string(); + let config_key_path = format!("{key_root}{CONFIG_KEY}"); + + let gen = quote! { + const DISCONFIG_KEY_ROOT: &'static str = #key_root; + + impl etcd_config::KeyHandler for #struct_name { + fn handle_key_updated(&self, key: &str, value: Option<&str>) { + match key { + #config_key_path => { + let proto = match etcd_config::base64_to_proto(value) { + Ok(Some(proto)) => proto, + Ok(None) => { + self.#field_name.store(std::sync::Arc::new(Default::default())); + tracing::info!("Resetting disconfig {} to default", #config_field_name); + + self.after_update(); + return; + } + Err(e) => { + tracing::error!("Failed to decode proto: {:?}", e); + return; + } + }; + + let new_proto = match Self::validate(proto) { + Ok(new_proto) => new_proto, + Err(e) => { + tracing::error!("Failed to validate proto: {:?}", e); + return; + } + }; + + self.#field_name.store(std::sync::Arc::new(new_proto)); + + self.after_update(); + } + _ => tracing::warn!("Unknown Disconfig entry: {} = {:?}", key, value), + } + } + } + + impl #struct_name { + pub async fn start_watching(disconfig: std::sync::Arc) -> anyhow::Result<()> { + use anyhow::Context; + + etcd_config::run_etcd_watcher(DISCONFIG_KEY_ROOT, disconfig.clone()) + .await + .context("Starting etcd watcher")?; + + Ok(()) + } + + } + }; + + Ok(gen.into()) + } + + fn fields(&self) -> &darling::ast::Fields> { + match &self.data { + darling::ast::Data::Enum(_) => unreachable!(), + darling::ast::Data::Struct(f) => f, + } + } +} + +#[derive(FromField, Debug)] +#[darling(forward_attrs(disconfig))] +pub(crate) struct DisconfigField { + pub(crate) ident: Option, + pub(crate) ty: syn::Type, +} + +#[derive(Debug, FromMeta)] +pub(crate) struct ConfigName { + name: SpannedValue, +} diff --git a/osprey_coordinator/etcd_config_derive/src/flags.rs b/osprey_coordinator/etcd_config_derive/src/flags.rs new file mode 100644 index 0000000..0bb1b5d --- /dev/null +++ b/osprey_coordinator/etcd_config_derive/src/flags.rs @@ -0,0 +1,207 @@ +use darling::{util::SpannedValue, FromDeriveInput, FromField, FromMeta}; +use quote::quote; + +#[derive(FromDeriveInput, Debug)] +#[darling(forward_attrs(service), supports(struct_any))] +pub(crate) struct FeatureFlags { + ident: syn::Ident, + attrs: Vec, + data: darling::ast::Data<(), SpannedValue>, +} + +#[derive(Debug)] +enum FieldType { + F32Type, + BoolType, + UsizeType, + U64Type, + U32Type, +} + +impl FeatureFlags { + pub(crate) fn impl_flags(&self) -> darling::Result { + let struct_name = &self.ident; + if self.attrs.len() > 1 { + return Err( + darling::Error::custom("Only one #[service] attribute is supported.") + .with_span(&self.attrs[1]), + ); + } + + let service_name_meta = &self + .attrs + .first() + .ok_or(darling::Error::custom("#[service] name must be set."))? + .meta; + let service_name = ServiceName::from_meta(service_name_meta)?.name.value(); + + let key_root = format!("/config/{service_name}"); + + let fields = self.fields(); + let mut handle_updates = Vec::with_capacity(fields.len()); + let mut check_methods = Vec::with_capacity(fields.len()); + + for field in fields.iter() { + let field_type = match &field.ty { + syn::Type::Path(type_path) if type_path.path.is_ident("AtomicF32") => { + FieldType::F32Type + } + syn::Type::Path(type_path) if type_path.path.is_ident("AtomicBool") => { + FieldType::BoolType + } + syn::Type::Path(type_path) if type_path.path.is_ident("AtomicUsize") => { + FieldType::UsizeType + } + syn::Type::Path(type_path) if type_path.path.is_ident("AtomicU64") => { + FieldType::U64Type + } + syn::Type::Path(type_path) if type_path.path.is_ident("AtomicU32") => { + FieldType::U32Type + } + _ => return Err(darling::Error::custom( + "Only field type AtomicF32, AtomicBool, AtomicUsize, AtomicU32 or AtomicU64 is supported.", + )), + }; + + let field_name = field.ident.clone().expect("field name must have ident."); + let field_name_str = field_name.to_string(); + + let config_key_path = format!("{key_root}/{field_name_str}"); + match field_type { + FieldType::BoolType => { + handle_updates.push(quote! { + #config_key_path => { + let new_value = etcd_config::convert_to_bool(value); + self.#field_name.store(new_value, std::sync::atomic::Ordering::Relaxed); + } + }); + check_methods.push(quote! { + pub fn #field_name(&self) -> bool { + self.#field_name.load(std::sync::atomic::Ordering::Relaxed) + } + }); + } + FieldType::F32Type => { + let mut as_bool = false; + for attr in &field.attrs { + let feature_flag_attrs = FeatureFlagAttributes::from_meta(&attr.meta)?; + as_bool = *feature_flag_attrs.as_bool; + } + + handle_updates.push(quote! { + #config_key_path => { + let new_value = etcd_config::convert_to_float(value); + self.#field_name.store(new_value); + } + }); + + // If has as_bool attribute, generate a method that returns bool. + if as_bool { + check_methods.push(quote! { + pub fn #field_name(&self) -> bool { + let value: f32 = ::rand::random(); + value < self.#field_name.load() + } + }); + } else { + check_methods.push(quote! { + pub fn #field_name(&self) -> f32 { + self.#field_name.load() + } + }); + } + } + FieldType::UsizeType | FieldType::U64Type | FieldType::U32Type => { + let (convert_to_value, return_type) = match field_type { + FieldType::UsizeType => ( + quote! { etcd_config::convert_to_usize(value) }, + quote! {usize}, + ), + FieldType::U64Type => { + (quote! { etcd_config::convert_to_u64(value) }, quote! {u64}) + } + FieldType::U32Type => { + (quote! { etcd_config::convert_to_u32(value) }, quote! {u32}) + } + _ => unreachable!(), + }; + + handle_updates.push(quote! { + #config_key_path => { + let new_value = #convert_to_value; + self.#field_name.store(new_value, std::sync::atomic::Ordering::Relaxed); + } + }); + check_methods.push(quote! { + pub fn #field_name(&self) -> #return_type { + self.#field_name.load(std::sync::atomic::Ordering::Relaxed) + } + }); + } + } + } + + let fields_updates = handle_updates + .into_iter() + .fold(quote! {}, |acc, line| quote! {#acc #line}); + let fields_checks = check_methods + .into_iter() + .fold(quote! {}, |acc, line| quote! {#acc #line}); + + let gen = quote! { + const CONFIG_KEY_ROOT: &str = #key_root; + + impl etcd_config::KeyHandler for #struct_name { + fn handle_key_updated(&self, key: &str, value: Option<&str>) { + match key { + #fields_updates + _ => tracing::warn!("Unknown EtcdConfig entry: {} = {:?}", key, value), + } + } + } + + impl #struct_name { + pub async fn start_watching() -> anyhow::Result> { + use anyhow::Context; + + let config = std::sync::Arc::new(Self::default()); + etcd_config::run_etcd_watcher(CONFIG_KEY_ROOT, config.clone()) + .await + .context("Starting etcd watcher")?; + + Ok(config) + } + + #fields_checks + } + }; + + Ok(gen.into()) + } + + fn fields(&self) -> &darling::ast::Fields> { + match &self.data { + darling::ast::Data::Enum(_) => unreachable!(), + darling::ast::Data::Struct(f) => f, + } + } +} + +#[derive(FromField, Debug)] +#[darling(forward_attrs(feature_flag))] +pub(crate) struct FeatureFlagField { + pub(crate) ident: Option, + pub(crate) ty: syn::Type, + pub(crate) attrs: Vec, +} + +#[derive(Debug, FromMeta)] +pub(crate) struct ServiceName { + name: SpannedValue, +} + +#[derive(Debug, FromMeta)] +pub(crate) struct FeatureFlagAttributes { + #[darling(default)] + as_bool: SpannedValue, +} diff --git a/osprey_coordinator/etcd_config_derive/src/lib.rs b/osprey_coordinator/etcd_config_derive/src/lib.rs new file mode 100644 index 0000000..9b5c677 --- /dev/null +++ b/osprey_coordinator/etcd_config_derive/src/lib.rs @@ -0,0 +1,54 @@ +extern crate proc_macro; +extern crate syn; +#[macro_use] +extern crate quote; + +use darling::FromDeriveInput; +use proc_macro::TokenStream; +use syn::{parse_macro_input, DeriveInput}; + +use crate::disconfigs::Disconfig; +use crate::flags::FeatureFlags; + +mod disconfigs; +mod flags; + +#[proc_macro_derive(FeatureFlags, attributes(service, feature_flag))] +pub fn derive_feature_flags(input: TokenStream) -> TokenStream { + fn feature_flags_inner( + derive_input: syn::DeriveInput, + ) -> darling::Result { + let feature_flags = FeatureFlags::from_derive_input(&derive_input)?; + let impls = feature_flags.impl_flags()?; + + Ok(quote! { + #impls + }) + } + + let derive_input = parse_macro_input!(input as DeriveInput); + match feature_flags_inner(derive_input) { + Ok(token_stream) => token_stream.into(), + Err(e) => TokenStream::from(e.write_errors()), + } +} + +#[proc_macro_derive(Disconfig, attributes(config))] +pub fn derive_disconfig(input: TokenStream) -> TokenStream { + fn disconfig_inner( + derive_input: syn::DeriveInput, + ) -> darling::Result { + let disconfig = Disconfig::from_derive_input(&derive_input)?; + let impls = disconfig.impl_config()?; + + Ok(quote! { + #impls + }) + } + + let derive_input = parse_macro_input!(input as DeriveInput); + match disconfig_inner(derive_input) { + Ok(token_stream) => token_stream.into(), + Err(e) => TokenStream::from(e.write_errors()), + } +} diff --git a/osprey_coordinator/metrics_derive/Cargo.toml b/osprey_coordinator/metrics_derive/Cargo.toml new file mode 100644 index 0000000..fd128d9 --- /dev/null +++ b/osprey_coordinator/metrics_derive/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "smite_coordinator_metrics_derive" +version = "0.1.0" +authors = ["osprey"] +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +darling = "0.20" +proc-macro2 = "1.0.46" +quote = "1.0.35" +syn = "2" diff --git a/osprey_coordinator/metrics_derive/src/lib.rs b/osprey_coordinator/metrics_derive/src/lib.rs new file mode 100644 index 0000000..6b6badf --- /dev/null +++ b/osprey_coordinator/metrics_derive/src/lib.rs @@ -0,0 +1,188 @@ +use darling::{ast::Data, util::SpannedValue, FromDeriveInput, FromField}; +use proc_macro2::{Literal, TokenStream, TokenTree}; +use quote::{quote, quote_spanned}; +use syn::{parse_macro_input, parse_quote, DeriveInput, GenericParam, Generics, TypeParamBound}; + +#[derive(Debug, FromDeriveInput)] +#[darling(attributes(metric))] +struct MetricsEmitterAttrs { + ident: syn::Ident, + generics: syn::Generics, + #[darling(default)] + prefix: String, + data: Data<(), SpannedValue>, +} + +#[derive(Debug, FromField)] +#[darling(attributes(metric))] +struct MetricsEmitterFieldAttrs { + ident: Option, + #[darling(default)] + name: String, +} + +#[proc_macro_derive(MetricsEmitter, attributes(metric))] +pub fn metrics_emitter_derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let input = match SpannedValue::::from_derive_input(&parse_macro_input!( + input as DeriveInput + )) { + Ok(input) => input, + Err(err) => return err.write_errors().into(), + }; + let name = &input.ident; + + // Add a bound `T: BaseMetric` to every type parameter T. + let generic_bounds: [TypeParamBound; 1] = [parse_quote!(crate::metrics::v2::BaseMetric)]; + let generics = add_trait_bounds(input.generics.clone(), generic_bounds); + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let emit_impl = implement_emit_metrics(&input); + + let expanded = quote! { + #[automatically_derived] + impl #impl_generics crate::metrics::v2::MetricsEmitter for #name #ty_generics #where_clause { + fn emit_metrics( + &self, + client: &crate::metrics::v2::macro_support::StatsdClient, + base_tags: &mut dyn ::std::iter::Iterator, + ) -> ::std::result::Result<(), crate::metrics::v2::macro_support::MetricError> { + #emit_impl + ::std::result::Result::Ok(()) + } + } + }; + + expanded.into() +} + +fn implement_emit_metrics(attrs: &SpannedValue) -> TokenStream { + match &attrs.data { + Data::Struct(data) => { + let recurse = data.fields.iter().enumerate().map(|(i, f)| { + let field_name: TokenTree = f.ident.as_ref().map(|id| id.clone().into()).unwrap_or_else(|| Literal::usize_unsuffixed(i).into()); + let mut name_string = String::new(); + if !attrs.prefix.is_empty() { + name_string.push_str(&attrs.prefix); + name_string.push('.'); + } + if !f.name.is_empty() { + name_string.push_str(&f.name); + } else { + name_string.push_str(&field_name.to_string()); + } + + let name_string = Literal::string(&name_string); + quote_spanned! { f.span() => + crate::metrics::v2::BaseMetric::send_metric(&self.#field_name, client, #name_string, &mut base_tags.iter().copied())?; + } + }); + quote! { + let base_tags: ::std::vec::Vec<(&str, &str)> = base_tags.collect(); + #(#recurse)* + } + } + Data::Enum(_) => { + darling::Error::custom("MetricsEmitter derive only implemented for structs") + .with_span(&attrs.span()) + .write_errors() + } + } +} + +#[derive(Debug, FromDeriveInput)] +#[darling(attributes(tag))] +struct TagKeyAttrs { + ident: syn::Ident, + generics: syn::Generics, + data: Data<(), SpannedValue>, +} + +#[derive(Debug, FromField)] +#[darling(attributes(tag))] +struct TagKeyFieldAttrs { + ident: Option, + #[darling(default)] + name: String, +} + +#[proc_macro_derive(TagKey, attributes(tag))] +pub fn tag_key_derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let input = match SpannedValue::::from_derive_input(&parse_macro_input!( + input as DeriveInput + )) { + Ok(input) => input, + Err(err) => return err.write_errors().into(), + }; + let name = &input.ident; + + // Add a bound `T: DynamicTagValue + Hash + Eq` to every type parameter T. + let generic_bounds: [TypeParamBound; 3] = [ + parse_quote!(crate::metrics::DynamicTagValue), + parse_quote!(::std::hash::Hash), + parse_quote!(::std::cmp::Eq), + ]; + let generics = add_trait_bounds(input.generics.clone(), generic_bounds); + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let (tag_key_impl, n) = implement_tag_key(&input); + + let expanded = quote! { + #[automatically_derived] + impl #impl_generics crate::metrics::v2::TagKey for #name #ty_generics #where_clause { + type TagIterator<'a> = ::std::array::IntoIter<(&'a str, &'a str), #n>; + fn tags<'a>( + &'a self, + ) -> Self::TagIterator<'a> { + #tag_key_impl + } + } + }; + + expanded.into() +} + +fn implement_tag_key(attrs: &SpannedValue) -> (TokenStream, usize) { + match &attrs.data { + Data::Struct(data) => { + let recurse = data.fields.iter().enumerate().map(|(i, f)| { + let field_name: TokenTree = f + .ident + .as_ref() + .map(|id| id.clone().into()) + .unwrap_or_else(|| Literal::usize_unsuffixed(i).into()); + let name_string = if f.name.is_empty() { + Literal::string(&field_name.to_string()) + } else { + Literal::string(&f.name) + }; + quote_spanned! { f.span() => + (#name_string, crate::metrics::DynamicTagValue::as_static_str(&self.#field_name)) + } + }); + let expanded = quote! { + [ + #(#recurse,)* + ].into_iter() + }; + (expanded, data.fields.len()) + } + Data::Enum(_) => ( + darling::Error::custom("TagKey derive only implemented for structs") + .with_span(&attrs.span()) + .write_errors(), + 0, + ), + } +} + +fn add_trait_bounds( + mut generics: Generics, + bounds: impl IntoIterator + Clone, +) -> Generics { + for param in &mut generics.params { + if let GenericParam::Type(ref mut type_param) = *param { + type_param.bounds.extend(bounds.clone()); + } + } + generics +} diff --git a/osprey_coordinator/src/backoff_utils/async_backoff.rs b/osprey_coordinator/src/backoff_utils/async_backoff.rs new file mode 100644 index 0000000..0124323 --- /dev/null +++ b/osprey_coordinator/src/backoff_utils/async_backoff.rs @@ -0,0 +1,36 @@ +use crate::backoff_utils::Config; +use backoff::{backoff::Backoff, exponential::ExponentialBackoff}; +use std::time::Duration; +use tokio::time::sleep; + +/// A futures safe backoff. +pub struct AsyncBackoff { + config: Config, + inner: ExponentialBackoff, +} + +impl AsyncBackoff { + pub fn new(config: Config) -> Self { + Self { + config: config.clone(), + inner: config.into(), + } + } + + /// Sleeps the current future. + pub async fn sleep(&mut self) { + let backoff_duration = self.fail(); + sleep(backoff_duration).await + } + + /// Returns the duration of the next backoff. Consumes the next backoff, and increases the `Duration` + /// that is returned the next time this function is called. + pub fn fail(&mut self) -> Duration { + self.inner.next_backoff().unwrap_or(self.config.max_delay) + } + + /// Resets the backoff. + pub fn succeed(&mut self) { + self.inner.reset() + } +} diff --git a/osprey_coordinator/src/backoff_utils/config.rs b/osprey_coordinator/src/backoff_utils/config.rs new file mode 100644 index 0000000..b251ac1 --- /dev/null +++ b/osprey_coordinator/src/backoff_utils/config.rs @@ -0,0 +1,46 @@ +use backoff::exponential::ExponentialBackoff; +use std::time::Duration; + +#[derive(Clone)] +pub struct Config { + /// The initial retry interval. + pub min_delay: Duration, + /// The maximum value of the back off period. Once the retry interval reaches this + /// value it stops increasing. + pub max_delay: Duration, + /// The value to multiply the current interval with for each retry attempt. + pub multiplier: f64, +} + +impl Config { + /// `Default` isn't `const` so we need this workaround, see: + /// https://stackoverflow.com/a/72467679 + pub const fn new_const_default() -> Self { + Self { + min_delay: Duration::from_secs(1), + max_delay: Duration::from_secs(60 * 5), + multiplier: 2.0, + } + } +} + +impl Default for Config { + fn default() -> Self { + Self::new_const_default() + } +} + +impl From for ExponentialBackoff { + fn from(val: Config) -> Self { + ExponentialBackoff { + initial_interval: val.min_delay, + multiplier: val.multiplier, + max_interval: val.max_delay, + // XXX: Setting this to Some(duration) will result in the backoff eventually + // failing. i.e. inner.next_backoff() will return None! + // Do not change this! + max_elapsed_time: None, + ..Default::default() + } + } +} diff --git a/osprey_coordinator/src/backoff_utils/mod.rs b/osprey_coordinator/src/backoff_utils/mod.rs new file mode 100644 index 0000000..a22e302 --- /dev/null +++ b/osprey_coordinator/src/backoff_utils/mod.rs @@ -0,0 +1,5 @@ +mod async_backoff; +mod config; + +pub use async_backoff::AsyncBackoff; +pub use config::Config; diff --git a/osprey_coordinator/src/cached_futures/mod.rs b/osprey_coordinator/src/cached_futures/mod.rs new file mode 100644 index 0000000..49de49b --- /dev/null +++ b/osprey_coordinator/src/cached_futures/mod.rs @@ -0,0 +1,193 @@ +use std::{collections::HashMap, fmt::Debug, future::Future, hash::Hash, sync::Arc}; + +use futures::{ + future::{BoxFuture, Shared}, + FutureExt, +}; +use parking_lot::RwLock; + +enum CachedFuture +where + T: Clone, + E: Clone, +{ + Ready(T), + Pending(Shared>>), +} + +pub enum CachedFutureRef<'a, T> { + Ready(&'a T), + Pending, +} + +#[derive(Clone)] +pub struct CachedFutures +where + K: Hash + Eq + Clone, + T: Clone, + E: Clone, +{ + cache: Arc>>>, +} + +impl Default for CachedFutures +where + K: Hash + Eq + Clone, + T: Clone, + E: Clone, +{ + fn default() -> Self { + Self { + cache: Default::default(), + } + } +} +impl Debug for CachedFutures +where + K: Hash + Eq + Clone, + T: Clone, + E: Clone, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CachedFutures") + .field("cache_size", &self.cache.read().len()) + .finish() + } +} + +impl CachedFutures +where + K: Hash + Eq + Clone + Send + Sync + 'static, + T: Clone + Send + Sync + 'static, + E: Clone + Send + Sync + 'static, +{ + /// Given a key, tries to re-use the successful result of `resolver`, otherwise, will execute resolver, + /// and cache its result if the result is Ok(). + pub async fn get_or_cache_default(&self, key: K, resolver: F) -> Result + where + F: FnOnce(K) -> U, + U: Future> + Send + 'static, + { + // Check to see if we already have a cached value - fast path with the read lock! + let maybe_pending_future = match self.cache.read().get(&key) { + Some(CachedFuture::Ready(value)) => return Ok(value.clone()), + Some(CachedFuture::Pending(future)) => Some(future.clone()), + None => None, + }; + + let future = match maybe_pending_future { + Some(future) => future, + None => { + // The future does not exist yet, so now we're going to acquire a write lock, so we can start resolution: + let mut write_locked_cache = self.cache.write(); + + // Repeat the read check first, a writer may have raced us and inserted the future: + match write_locked_cache.get(&key) { + Some(CachedFuture::Ready(value)) => return Ok(value.clone()), + Some(CachedFuture::Pending(future)) => future.clone(), + None => { + let future = self.create_resolver_future(resolver, &key); + write_locked_cache.insert(key, CachedFuture::Pending(future.clone())); + future + } + } + } + }; + + future.await + } + + /// Reduces over the cached futures, allowing one to introspect all of the resolved future values, or which futures are pending. + pub fn fold(&self, mut init: I, reducer: F) -> I + where + F: Fn(I, (&K, CachedFutureRef<'_, T>)) -> I, + { + let cache = self.cache.read(); + for (k, v) in cache.iter() { + let future_ref = match v { + CachedFuture::Ready(v) => CachedFutureRef::Ready(v), + CachedFuture::Pending(_) => CachedFutureRef::Pending, + }; + + init = (reducer)(init, (k, future_ref)); + } + + init + } + + /// Reduces over the ready cached futures, allowing one to introspect all of the resolved future values. + pub fn fold_ready(&self, init: I, reducer: F) -> I + where + F: Fn(I, (&K, &T)) -> I, + { + self.fold(init, |init, (k, v)| match v { + CachedFutureRef::Ready(v) => (reducer)(init, (k, v)), + CachedFutureRef::Pending => init, + }) + } + + // drop a cached future + pub fn drop(&self, key: &K) { + self.cache.write().remove(key); + } + + /// Iterates over all the *ready* cached futures, retaining ones that are match the predicate function, and returning the number + /// of futures which have been disposed of (where the predicate function has returned false). + pub fn retain_ready(&self, mut f: F) -> usize + where + F: FnMut(&T) -> bool, + { + let mut cache = self.cache.write(); + let mut num_disposed = 0; + cache.retain(|_, v| { + match v { + // Always retain pending entries, as they are in-flight. + CachedFuture::Pending(_) => true, + CachedFuture::Ready(r) => { + let should_retain = (f)(r); + + if !should_retain { + num_disposed += 1; + } + + should_retain + } + } + }); + + num_disposed + } + + fn create_resolver_future( + &self, + resolver: F, + key: &K, + ) -> Shared>> + where + F: FnOnce(K) -> U, + U: Future> + Send + 'static, + { + let resolver_future = (resolver)(key.clone()); + let cache = self.cache.clone(); + let key_clone = key.clone(); + async move { + let result = resolver_future.await; + match result { + Ok(result) => { + // If the result was a success, we'll cache that result now. + cache + .write() + .insert(key_clone, CachedFuture::Ready(result.clone())); + Ok(result) + } + Err(error) => { + // We will not cache errors! + cache.write().remove(&key_clone); + Err(error) + } + } + } + .boxed() + .shared() + } +} diff --git a/osprey_coordinator/src/coordinator_metrics.rs b/osprey_coordinator/src/coordinator_metrics.rs new file mode 100644 index 0000000..8bf7efb --- /dev/null +++ b/osprey_coordinator/src/coordinator_metrics.rs @@ -0,0 +1,66 @@ +use crate::metrics::define_metrics; + +define_metrics!(SmiteCoordinatorMetrics, [ + // How long an action has been processed by a worker for before it is acked/nacked + action_outstanding_duration => StaticHistogram(), + // How long an action has been held in the async queue for before it is sent to a worker + action_time_in_async_queue => StaticHistogram("action_time_in_queue", ["async" => "true"]), + // How many new connections have been established to the coordinator + new_connection_established => StaticCounter(), + + // How many messages are currently buffered in the priority queue + priority_queue_size_sync => StaticGauge("priority_queue_size",["type" => "sync"]), + priority_queue_size_async => StaticGauge("priority_queue_size",["type" => "async"]), + + // How many receivers are open for the priority queue + // can be used as a proxy for number of connections open from the smite worker + priority_queue_receiver_count_sync => StaticGauge("priority_queue_receiver_count",["type" => "sync"]), + priority_queue_receiver_count_async => StaticGauge("priority_queue_receiver_count",["type" => "async"]), + + client_disconnected_gracefully => StaticCounter("client_disconnected", ["error" => "false"]), + client_disconnected_broken_pipe => StaticCounter("client_disconnected", ["error" => "broken_pipe"]), + client_disconnected_stream_error => StaticCounter("client_disconnected", ["error" => "stream_error"]), + client_disconnected_receiver_closed => StaticCounter("client_disconnected", ["error" => "pq_closed"]), + client_disconnected_receiver_timeout => StaticCounter("client_disconnected", ["error" => "pq_timeout"]), + + bidi_stream_ended_no_outstanding_action => StaticCounter("bidi_stream_ended", ["outstanding_action" => "false"]), + bidi_stream_ended_yes_outstanding_action => StaticCounter("bidi_stream_ended", ["outstanding_action" => "true"]), + + // How many sync actions have been recieved + sync_classification_action_received => StaticCounter(), + + // How many sync actions have failed due to deserialization issues + sync_classification_failure_deserialization => StaticCounter("sync_classification_failure", ["error" => "deserialization"]), + // How many sync actions have failed due to the acking oneshot being dropped + sync_classification_failure_oneshot_dropped => StaticCounter("sync_classification_failure", ["error" => "oneshot_dropped"]), + // How many sync actions have failed because we couldn't send to the priority queue + sync_classification_failure_pq_send => StaticCounter("sync_classification_failure", ["error" => "pq_send"]), + sync_classification_failure_label_service => StaticCounter("sync_classification_failure", ["error" => "label_service"]), + + // How many acks for sync actions + sync_classification_result_ack => StaticCounter("sync_classification_result", ["ack" => "true"]), + // How many nacks for sync actions + sync_classification_result_nack => StaticCounter("sync_classification_result", ["ack" => "false"]), + + // How many messages we've added to the async queue + async_classification_added_to_queue => StaticCounter("async_classification_added_to_queue"), + // How many acks for async actions + async_classification_result_ack => StaticCounter("async_classification_result", ["ack" => "true"]), + // How many nacks for async actions + async_classification_result_nack => StaticCounter("async_classification_result", ["ack" => "false"]), + + // Time it took to send a message to the Async-Channel priority queue + priority_queue_send_time_async => StaticHistogram("priority_queue_send_time", ["type" => "async"]), + // Time it took to send a message to the sync-Channel priority queue + priority_queue_send_time_sync => StaticHistogram("priority_queue_send_time", ["type" => "sync"]), + // Time it took to receive a message from the async/sync priority queue + priority_queue_receive_time => StaticHistogram("priority_queue_receive_time"), + // Time it took for the acking receiver in the MessageHandler to receive an ack/nack + receiver_ack_time_async => StaticHistogram("receiver_ack_time", ["type" => "async"]), + // Time it took for the acking receiver in the MessageHandler to receive an ack/nack + receiver_ack_time_sync => StaticHistogram("receiver_ack_time", ["type" => "sync"]), + + // How many times action ID generation from snowflake is used (when pubsub_action.id is None) + action_id_snowflake_generation_json => StaticCounter("action_id_snowflake_generation", ["proto"=>"false"]), + action_id_snowflake_generation_proto => StaticCounter("action_id_snowflake_generation", ["proto"=>"true"]), +]); diff --git a/osprey_coordinator/src/discovery/announcer.rs b/osprey_coordinator/src/discovery/announcer.rs new file mode 100644 index 0000000..a2d1aff --- /dev/null +++ b/osprey_coordinator/src/discovery/announcer.rs @@ -0,0 +1,255 @@ +use crate::backoff_utils::{AsyncBackoff, Config as BackoffConfig}; +use crate::discovery::directory; +use crate::discovery::service::ServiceRegistration; +use crate::etcd::Client; +use anyhow::Result; +use futures::future::FutureExt; +use log::{debug, error, info, warn}; +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::oneshot; +use tokio::task::{spawn, JoinHandle}; + +const DEFAULT_TTL: Duration = Duration::from_secs(30); +const SHUTDOWN_RETRY_ATTEMPTS: usize = 3; + +#[derive(Clone, Debug)] +struct Config { + /// Registration that should be announced. + registration: ServiceRegistration, + /// Duration the service should stay announced before automatically falling out of service + /// discovery. + ttl: Duration, + /// Etcd client. + client: Arc, + /// Whether or not to wait for the key to be free before registering. Defaults to `false` if the `Option` is `None` + wait_for_free_key_timeout: Option, +} + +/// Builder for a [`ServiceAnnouncer`]. +#[derive(Clone, Debug)] +pub struct Builder { + config: Config, +} + +impl Builder { + fn new(registration: ServiceRegistration, client: Arc) -> Self { + Builder { + config: Config { + registration, + ttl: DEFAULT_TTL, + client, + wait_for_free_key_timeout: None, + }, + } + } + + pub fn wait_for_free_key(mut self, timeout: Option) -> Self { + self.config.wait_for_free_key_timeout = timeout; + self + } + + pub fn with_ttl(mut self, ttl: Duration) -> Self { + self.config.ttl = ttl; + self + } + + /// Start the announcer. + /// It will continue announcing until [`ServiceAnnouncer.stop`] is called. + /// + /// [`ServiceAnnouncer.stop`]: struct.ServiceAnnouncer.html#stop + pub fn start(self) -> ServiceAnnouncer { + info!("Starting announcer."); + let (stop_tx, stop_rx) = oneshot::channel(); + let (drain_tx, drain_rx) = oneshot::channel(); + + let join_handle = spawn(async move { + announce(self.config, stop_rx, drain_rx).await; + }); + + ServiceAnnouncer { + state: Some(State { + stop_tx, + drain_tx: Some(drain_tx), + join_handle, + }), + } + } +} + +/// A service announcer used to automatically register a service in discovery at an interval. +/// Dropping the `ServiceAnnouncer` will send a shutdown signal to the announcement task, +/// but in most cases you should call [`stop`]. +/// +/// [`stop`]: ServiceAnnouncer::stop() +#[derive(Debug)] +pub struct ServiceAnnouncer { + state: Option, +} + +#[derive(Debug)] +struct State { + /// Channel for sending a stop message to the announcer. + stop_tx: oneshot::Sender<()>, + /// Channel for sending a drain message to the announcer. + drain_tx: Option>, + + join_handle: JoinHandle<()>, +} + +impl ServiceAnnouncer { + #[inline] + pub fn builder>>(registration: ServiceRegistration, client: C) -> Builder { + Builder::new(registration, client.into()) + } + + /// Start a new `Announcer` for a service with defaults. + /// Errors if `ETCD_PEERS` is not set in the environment. + pub fn start(registration: ServiceRegistration) -> Result { + Ok(Self::builder(registration, Arc::new(Client::from_etcd_peers()?)).start()) + } + + /// Send a drain signal to the announcement task + pub fn mark_as_draining(&mut self) { + if let Some(state) = &mut self.state { + if let Some(drain_tx) = state.drain_tx.take() { + info!("Notifying task to set drain state to true."); + drain_tx.send(()).ok(); + } + } + } + + /// Send a shutdown signal to the announcement task, + /// then returns a future of the task's completion. + pub fn stop(mut self) -> impl Future { + // We specifically don't want this to be an async function + // because we want the interrupt to happen immediately. + // An async function would defer the interrupt to the first poll. + + info!("Stopping announcer."); + let state = self.state.take().unwrap(); + // If this fails then the announcer is already stopped. + state.stop_tx.send(()).ok(); + state.join_handle.map(|_| ()) + } +} + +impl Drop for ServiceAnnouncer { + fn drop(&mut self) { + if let Some(State { stop_tx, .. }) = self.state.take() { + stop_tx.send(()).ok(); + } + } +} + +/// Core loop function that continually announces. +/// +/// # Arguments +/// +/// * `config` - Announce configuration. +/// * `stop_rx` - receiver that listens for stop messages. +/// * `drain_rx` - receiver that listens for drain messages. +async fn announce( + mut config: Config, + stop_rx: oneshot::Receiver<()>, + drain_rx: oneshot::Receiver<()>, +) { + let client = &config.client; + let registration = &mut config.registration; + let ttl = config.ttl; + + debug!("Starting announce service: {:?}", registration); + + let mut backoff = AsyncBackoff::new(BackoffConfig { + min_delay: Duration::from_secs(1), + max_delay: ttl / 2, + ..Default::default() + }); + + let mut refresh = false; + + let mut stop_rx = stop_rx.fuse(); + let mut drain_rx = drain_rx.fuse(); + loop { + if refresh { + match directory::refresh(client, registration, ttl).await { + Ok(()) => { + backoff.succeed(); + } + // If the refresh request fails, we should retry the register, setting + // refresh to false. + Err(e) => { + warn!("Failure to refresh. error={}", e); + // If we are dealing with a timeout error, we can still try and refresh, we'll just backoff a bit, then attempt to refresh! + // We *only* need to re-register if we encounter a "key not found" error. + if let Ok(err) = e.downcast::() { + if matches!(err, crate::etcd::EtcdError::IoError { .. }) { + backoff.sleep().await; + continue; + } + } + + refresh = false; + continue; + } + } + } else { + if !registration.draining { + if let Some(timeout) = config.wait_for_free_key_timeout { + debug!( + "Waiting for free key to announce service: {:?}", + registration + ); + match directory::wait_for_free_key(client, ®istration.key(), timeout).await { + Ok(()) => { + debug!("Free key found for service: {:?}", registration); + } + Err(_) => { + warn!( + "Free key not found within the allotted timeout for service: {:?}", + registration + ); + backoff.sleep().await; + continue; + } + } + } + } + debug!("Announcing service: {:?}", registration); + match directory::register(client, registration, ttl).await { + Ok(()) => { + info!("Announced service: {:?}", registration); + refresh = true; + backoff.succeed(); + } + Err(e) => { + warn!("Failure to set. error={}", e); + backoff.sleep().await; + } + } + } + + let delay = tokio::time::sleep((ttl / 10) * 4).fuse(); + tokio::select! { + _ = delay => {}, + _ = &mut stop_rx => { + debug!("Stopping announce service: {:?}", registration); + for attempt in 0..SHUTDOWN_RETRY_ATTEMPTS { + if let Err(e) = directory::unregister(client, registration).await { + error!("Uannounce attempt {}: Failed to unregister. {}", attempt, e); + backoff.sleep().await; + } else { + info!("Unannounced service: {:?}", registration); + return; + } + } + } + _ = &mut drain_rx => { + registration.draining = true; + refresh = false; + debug!("Reanouncing service as draining: {:?}", registration); + } + }; + } +} diff --git a/osprey_coordinator/src/discovery/directory.rs b/osprey_coordinator/src/discovery/directory.rs new file mode 100644 index 0000000..7a0db17 --- /dev/null +++ b/osprey_coordinator/src/discovery/directory.rs @@ -0,0 +1,90 @@ +use crate::discovery::service::ServiceRegistration; +use crate::etcd::{Client, EtcdError}; +use anyhow::Result; +use log::trace; +use std::time::Duration; +use tokio::time::error::Elapsed; + +const WAIT_FOR_FREE_KEY_RETRY_INTERVAL: Duration = Duration::from_secs(2); + +/// Waits for a given key to become free from service discovery. +/// An upper timeout is required since this method runs indefinitely otherwise. +/// +/// # Arguments +/// +/// * `client` - the etcd client to use +/// * `key` - the service key to wait for to become free +/// * `timeout` - the `Duration` after which the method will return an error if the key is not freed +pub async fn wait_for_free_key( + client: &Client, + key: &str, + timeout: Duration, +) -> Result<(), Elapsed> { + tokio::time::timeout(timeout, async move { + loop { + let response = client.get(key).await; + match response { + Ok(response) => { + if response.data.node.value.is_none() { + // key is *basically* free („• ᴗ •„) + return; + } + } + Err(err) => match err { + EtcdError::KeyNotFound { index: _, error: _ } => { + return; + } + _ => {} + }, + } + tokio::time::sleep(WAIT_FOR_FREE_KEY_RETRY_INTERVAL).await; + } + }) + .await +} + +/// Registers a service in service discovery. +/// +/// # Arguments +/// +/// * `client` - the etcd client to use +/// * `service` - the service to register +/// * `ttl` - the `Duration` for the service to stay in discovery until it automatically falls +/// falls out. +pub async fn register(client: &Client, service: &ServiceRegistration, ttl: Duration) -> Result<()> { + trace!("Registering {:?}", service); + let key = service.key(); + Ok(client + .set(&*key, service, Some(ttl.as_secs())) + .await + .map(|_response| ())?) +} + +/// Refreshes a service registration in discovery. +/// +/// # Arguments +/// +/// * `client` - the etcd client to use +/// * `service` - the service to register +/// * `ttl` - the `Duration` for the service to stay in discovery until it automatically falls +/// falls out. +pub async fn refresh(client: &Client, service: &ServiceRegistration, ttl: Duration) -> Result<()> { + trace!("Refreshing {:?}", service); + let key = service.key(); + Ok(client + .refresh(&*key, ttl.as_secs()) + .await + .map(|_response| ())?) +} + +/// Unregister a service from discovery. +/// +/// # Arguments +/// +/// * `client` - the etcd client to use +/// * `service` - the service to unregister +pub async fn unregister(client: &Client, service: &ServiceRegistration) -> Result<()> { + trace!("Unregistering {:?}", service); + let key = &*service.key(); + Ok(client.delete(key).await?) +} diff --git a/osprey_coordinator/src/discovery/error.rs b/osprey_coordinator/src/discovery/error.rs new file mode 100644 index 0000000..309c3f3 --- /dev/null +++ b/osprey_coordinator/src/discovery/error.rs @@ -0,0 +1,15 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum DiscoveryError { + #[error("Response has no index")] + NoIndex, + #[error("Node has no value")] + NoNodeValue, + #[error("Failed to resolve hostname to ip. hostname={}", hostname)] + FailedToResolveHostname { hostname: String }, + #[error("No ring configured but select_key called.")] + NoRingConfigured, + #[error("No services in discovery.")] + NoServices, +} diff --git a/osprey_coordinator/src/discovery/lib.rs b/osprey_coordinator/src/discovery/lib.rs new file mode 100644 index 0000000..0d046a2 --- /dev/null +++ b/osprey_coordinator/src/discovery/lib.rs @@ -0,0 +1,21 @@ +#[macro_use] +extern crate log; + +mod announcer; +#[cfg(not(feature = "integration_test"))] +mod directory; +#[cfg(feature = "integration_test")] +pub mod directory; +mod error; +mod limits; +mod ring; +mod service; +mod types; +mod watcher; + +pub use announcer::ServiceAnnouncer; +pub use error::DiscoveryError; +pub use limits::{GlobalLimiter, Member, Normal}; +pub use ring::{Config as RingConfig, Ring}; +pub use service::ServiceRegistration; +pub use watcher::ServiceWatcher; diff --git a/osprey_coordinator/src/discovery/limits.rs b/osprey_coordinator/src/discovery/limits.rs new file mode 100644 index 0000000..f9ac94d --- /dev/null +++ b/osprey_coordinator/src/discovery/limits.rs @@ -0,0 +1,91 @@ +use crate::discovery::service::ServiceRegistration; +use crate::discovery::watcher::ServiceWatcher; +use crate::etcd::Client; +use anyhow::Result; + +#[derive(Clone, Debug)] +pub struct GlobalLimiter { + /// service watcher to keep track of cluster size + service_watcher: ServiceWatcher, + /// Limiter type is either [`Normal`] or [`Member`]. + limiter_type: T, +} + +// type used for unowned limiter, cannot determine self share +#[derive(Clone, Debug)] +pub struct Normal; + +// type used for owned limiter, can determine self share +#[derive(Clone, Debug)] +pub struct Member { + /// service id for this node + self_id: String, +} + +impl GlobalLimiter { + /// Watch a service's instances for the calculation of per-node limits. + /// + /// # Arguments + /// + /// * `name` - the service name + /// * `client` - etcd client + /// * `ring` - wether the service uses a ring config + pub async fn watch>(name: N, client: Client, ring: bool) -> Result { + let service_watcher = ServiceWatcher::watch(name, client, ring).await?; + + let global_limiter = Self { + service_watcher, + limiter_type: Normal, + }; + + Ok(global_limiter) + } +} + +impl GlobalLimiter { + /// Watch this service's instances for the calculation of per-node limits, implies that a ring config is used. + /// + /// # Arguments + /// + /// * `self_registration` - this node's service registration + /// * `client` - etcd client + pub async fn watch_as_member( + self_registration: &ServiceRegistration, + client: Client, + ) -> Result { + let service_watcher = + ServiceWatcher::watch(self_registration.name.clone(), client, true).await?; + + let global_limiter = Self { + service_watcher, + limiter_type: Member { + self_id: self_registration.id(), + }, + }; + + Ok(global_limiter) + } + + /// Get this node's share, based on the amount of the ring owned by this node + pub fn get_self_share(&self, global_limit: f64) -> Option { + self.get_node_share(global_limit, &self.limiter_type.self_id) + } +} + +impl GlobalLimiter { + /// Get the equal share, based on the number of instances + pub fn get_equal_share(&self, global_limit: f64) -> Option { + match self.service_watcher.num_instances() { + num_instances if num_instances > 0 => Some(global_limit / (num_instances as f64)), + _ => None, + } + } + + /// Get the given node's share, based on the amount of the ring owned by that node + pub fn get_node_share(&self, global_limit: f64, service_id: impl AsRef) -> Option { + match self.service_watcher.ring_share(service_id) { + ring_share if ring_share > 0.0 => Some(global_limit * ring_share), + _ => None, + } + } +} diff --git a/osprey_coordinator/src/discovery/mod.rs b/osprey_coordinator/src/discovery/mod.rs new file mode 100644 index 0000000..a26ea22 --- /dev/null +++ b/osprey_coordinator/src/discovery/mod.rs @@ -0,0 +1,18 @@ +mod announcer; +#[cfg(not(feature = "integration_test"))] +mod directory; +#[cfg(feature = "integration_test")] +pub mod directory; +mod error; +mod limits; +mod ring; +mod service; +mod types; +mod watcher; + +pub use announcer::ServiceAnnouncer; +pub use error::DiscoveryError; +pub use limits::{GlobalLimiter, Member, Normal}; +pub use ring::{Config as RingConfig, Ring}; +pub use service::ServiceRegistration; +pub use watcher::ServiceWatcher; diff --git a/osprey_coordinator/src/discovery/ring.rs b/osprey_coordinator/src/discovery/ring.rs new file mode 100644 index 0000000..3d0f8a9 --- /dev/null +++ b/osprey_coordinator/src/discovery/ring.rs @@ -0,0 +1,431 @@ +#[cfg(feature = "modify")] +use std::num::NonZeroU32; +use std::{collections::HashMap, sync::Arc}; + +use crate::etcd::{Action, Client, KeyValueInfo, WatchEvent, Watcher}; +use crate::hashring::{HashRing, Node as HashRingNode}; +use crate::tokio_utils::AbortOnDrop; +use anyhow::Result; +use arc_swap::ArcSwap; +use log::{debug, error, info, warn}; +use tokio::{ + sync::oneshot, + task::{block_in_place, spawn}, +}; +use tokio_stream::StreamExt; + +use crate::discovery::{ + types::ring::{RingConfig, RingConfigV1}, + DiscoveryError, +}; + +pub(crate) const DEFAULT_NUM_REPLICAS: u32 = 512; + +/// A ring of services used to consistently find a service instance in discovery by a key. +#[derive(Clone)] +pub struct Ring { + inner: Arc, + + /// When all references are dropped, will terminate the ring watcher. + _abort_on_drop: Arc>, +} + +impl std::fmt::Debug for Ring { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Ring").field("inner", &self.inner).finish() + } +} + +#[derive(Debug)] +struct RingInner { + /// Mutable ring data. + data: ArcSwap, + /// The key in discovery where the ring will be stored. + key: String, + /// The ring's config. + config: Config, +} + +#[derive(Debug, Default)] +struct RingData { + /// Internal hash ring. + ring: HashRing, + /// Ring key overrides + overrides: HashMap>, +} + +#[derive(Debug)] +pub struct Config { + /// Etcd client for setting the ring. + pub client: Client, + /// Default num replicas if the node doesn't specify. + pub default_num_replicas: u32, +} + +impl Config { + pub fn from_environment() -> Result { + Ok(Self { + client: Client::from_etcd_peers()?, + default_num_replicas: DEFAULT_NUM_REPLICAS, + }) + } +} + +impl Ring { + /// Create a new `Ring`. Watches the key '/discovery//ring'. + /// + /// # Arguments + /// + /// * `name` - the name of the service, used to construct the watched key + /// * `client` - etcd client + pub async fn new(name: String) -> Result { + let config = Config::from_environment()?; + Self::from_config(name, config).await + } + + /// Create a new `Ring` with a specific number of replicas per node. + /// + /// # Arguments + /// + /// * `name` - the name of the service, used to construct the watched key + /// * `config` - ring `Config` + pub async fn from_config(name: String, config: Config) -> Result { + let key = format!("/discovery/{}/ring", name); + Self::from_config_with_key(key, config).await + } + + /// Create a new `Ring` with a specific number of replicas per node, + /// specifying the actual key to watch. + /// + /// # Arguments + /// + /// * `name` - the name of the service + /// * `key` - the key to watch for ring updates + /// * `config` - ring `Config` + pub async fn from_config_with_key(key: String, config: Config) -> Result { + let inner = Arc::new(RingInner { + key, + config, + data: Default::default(), + }); + + let (ready_tx, ready_rx) = oneshot::channel(); + let join_handle = spawn(watch(inner.clone(), ready_tx)); + + let ring = Ring { + inner, + _abort_on_drop: Arc::new(AbortOnDrop::new(join_handle)), + }; + + ready_rx.await?; + Ok(ring) + } + + /// Select a single instance from discovery based on a key. + /// + /// # Arguments + /// + /// * `key` - the key to select for + pub fn select(&self, key: impl AsRef) -> Option> { + let data = self.inner.data.load(); + let key = key.as_ref(); + + if let Some(node) = data.overrides.get(key) { + return Some(node.clone()); + } + + data.ring.find(key).ok().cloned() + } + + /// Select multiple instances from discovery based on a key. + /// + /// # Arguments + /// + /// * `key` - the key to select for + /// * `n` - the number of instances to select, if an override exists return value could be above this limit + pub fn select_n(&self, key: impl AsRef, n: usize) -> Option>> { + let data = self.inner.data.load(); + let key = key.as_ref(); + let nodes = data.ring.find_n(key, n).ok(); + + match data.overrides.get(key) { + // No override, so we don't need to fiddle with the returned nodes array. + None => nodes, + // We have an override, so we'll set that override to the first position returned. + Some(override_node) => match nodes { + Some(mut nodes) => { + nodes.retain(|node_name| node_name != override_node); + nodes.insert(0, override_node.clone()); + Some(nodes) + } + // No matches in the ring, but we've got an override. + None => Some(vec![override_node.clone()]), + }, + } + } + + /// Returns the members of the ring. + pub fn nodes(&self) -> Vec { + self.inner.data.load().ring.nodes().collect_nodes() + } + + /// Adds a single node to the ring. Warning: This is supposed to be a manual operation + /// completed by a human operator. If this is done automatically then some data may be lost. + /// + /// # Arguments + /// + /// * `name` - name of the instances. Normally hostname:port. + /// * `num_replicas` - the number of replicas for the instance + #[cfg(feature = "modify")] + pub async fn add(&self, name: impl Into>, num_replicas: NonZeroU32) -> Result<()> { + let name = name.into(); + debug!("Add {:?} {:?}", name, num_replicas); + + let mut nodes = self.inner.data.load().ring.nodes().clone(); + nodes.add(HashRingNode::new(name, num_replicas.get()))?; + self.persist_nodes(nodes).await + } + + /// Removes a single node from the ring. Warning: This is supposed to be a manual operation + /// completed by a human operator. If this is done automatically then some data may be lost. + /// + /// # Arguments + /// + /// * `name` - name of the instances. Normally hostname:port. + #[cfg(feature = "modify")] + pub async fn remove(&self, name: impl AsRef) -> Result<()> { + let node_name = name.as_ref(); + debug!("Remove {:?}", node_name); + + let mut nodes = self.inner.data.load().ring.nodes().clone(); + nodes.remove(node_name)?; + self.persist_nodes(nodes).await + } + + /// Saves the members of the `Ring` to etcd. + #[cfg(feature = "modify")] + pub async fn persist_nodes(&self, nodes: hashring::RingNodes) -> Result<()> { + let nodes = nodes.collect_nodes(); + match self + .inner + .config + .client + .set(&*self.inner.key, nodes, None) + .await + { + Ok(_response) => Ok(()), + Err(e) => Err(e.into()), + } + } +} + +/// Watches etcd for changes to the ring. +async fn watch(ring: Arc, ready_tx: oneshot::Sender<()>) { + let mut watcher = Watcher::new(ring.config.client.clone(), ring.key.clone()); + let mut ready_tx = Some(ready_tx); + + loop { + let watch_event = match watcher.next().await { + Some(event) => event, + None => return, + }; + + let maybe_members = match watch_event { + WatchEvent::Get(None) => { + info!("Initialized with no nodes."); + Some(RingConfigV1::default()) + } + WatchEvent::Get(Some(info)) | WatchEvent::Update(info) => { + get_ring_config(info, ring.config.default_num_replicas) + } + }; + + let new_config = match maybe_members { + Some(new_config) => new_config, + None => continue, + }; + + debug!("Updating ring config: {:?}", new_config); + + // Rebuilding the ring can be "expensive", so we'll hint to the tokio executor that we want to run this in the blocking executor to avoid + // blocking the runtime for other tasks. + block_in_place(|| { + let RingConfigV1 { overrides, members } = new_config; + let ring_data = RingData { + ring: HashRing::with_nodes(members), + overrides: overrides + .unwrap_or_default() + .into_iter() + .map(|(key, node_name)| { + ( + // We need to convert the key to string because rust uses a string while the + // current configs use u64. + key.0.to_string(), + node_name.0.into(), + ) + }) + .collect(), + }; + // Atomically update ring data. + ring.data.store(Arc::new(ring_data)); + }); + + // Notify waiter that we've read the ring. + if let Some(ready_tx) = ready_tx.take() { + ready_tx.send(()).ok(); + } + } +} + +/// Get the members from an etcd watch update. +/// +/// # Arguments +/// +/// * info - etcd node info +/// * default_num_replicas - the default number of replicas for a node +fn get_ring_config(info: KeyValueInfo, default_num_replicas: u32) -> Option { + match info.action { + Action::Get | Action::Set | Action::Create | Action::CompareAndSwap | Action::Update => { + // Deserializes the members from an etcd node. An etcd node can have two types of + // data for members due to legacy reasons. + let config = match &info.node.value { + None => Err(DiscoveryError::NoNodeValue.into()), + Some(value) => deserialize_config(value, default_num_replicas), + }; + + match config { + Ok(new_config) => Some(new_config), + Err(e) => { + error!("Failed to deserialize members. {}", e); + None + } + } + } + Action::Delete | Action::Expire | Action::CompareAndDelete => { + warn!("All members offline."); + Some(RingConfigV1::default()) + } + } +} + +fn deserialize_config(value: &str, default_num_replicas: u32) -> Result { + let ring_config = serde_json::from_str::(value)?; + Ok(ring_config.convert_to_v1(default_num_replicas)) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn assert_nodes(mut actual_nodes: Vec, mut expected_nodes: Vec) { + actual_nodes.sort(); + expected_nodes.sort(); + + assert_eq!(actual_nodes, expected_nodes); + } + + #[test] + fn it_deserializes_structs() { + let expected_nodes = vec![ + HashRingNode::new("localhost:3000", 10), + HashRingNode::new("localhost:3001", 20), + ]; + let value = json!([ + {"name": expected_nodes[0].name(), "num_replicas": expected_nodes[0].num_replicas()}, + {"name": expected_nodes[1].name(), "num_replicas": expected_nodes[1].num_replicas()} + ]) + .to_string(); + let config = deserialize_config(&value, 30).expect("failed to deserialize"); + assert_nodes(config.members, expected_nodes); + } + + #[test] + fn it_deserializes_strings() { + let expected_nodes = vec![ + HashRingNode::new("localhost:3000", 20), + HashRingNode::new("localhost:3001", 20), + ]; + let value = json!([expected_nodes[0].name(), expected_nodes[1].name()]).to_string(); + let config = deserialize_config(&value, 20).expect("failed to deserialize"); + assert_nodes(config.members, expected_nodes); + } + + #[test] + fn it_deserializes_mixed() { + let expected_nodes = vec![ + HashRingNode::new("localhost:3000", 20), + HashRingNode::new("localhost:3001", 10), + ]; + let value = json!([ + expected_nodes[0].name(), + {"name": expected_nodes[1].name(), "num_replicas": expected_nodes[1].num_replicas()} + ]) + .to_string(); + + let config = deserialize_config(&value, 20).expect("failed to deserialize"); + assert_nodes(config.members, expected_nodes); + } + + #[test] + fn it_deserializes_v1() { + let expected_nodes = vec![ + HashRingNode::new("localhost:3000", 20), + HashRingNode::new("localhost:3001", 10), + ]; + + let value = json!({ + "schema": "v1", + "members": [ + {"name": expected_nodes[0].name(), "num_replicas": expected_nodes[0].num_replicas()}, + {"name": expected_nodes[1].name(), "num_replicas": expected_nodes[1].num_replicas()} + ] + }) + .to_string(); + + let config = deserialize_config(&value, 20).expect("failed to deserialize"); + assert_nodes(config.members, expected_nodes); + assert!(config.overrides.is_none()); + } + + #[test] + fn it_deserializes_v1_overrides() { + let expected_nodes = vec![ + HashRingNode::new("localhost:3000", 20), + HashRingNode::new("localhost:3001", 10), + ]; + + let expected_overrides = vec![(1234u64, &expected_nodes[0]), (5678u64, &expected_nodes[1])]; + + let value = json!({ + "schema": "v1", + "overrides": [ + [expected_overrides[0].0, expected_overrides[0].1.name()], + [expected_overrides[1].0, expected_overrides[1].1.name()] + ], + "members": [ + {"name": expected_nodes[0].name(), "num_replicas": expected_nodes[0].num_replicas()}, + {"name": expected_nodes[1].name(), "num_replicas": expected_nodes[1].num_replicas()} + ] + }) + .to_string(); + + let config = deserialize_config(&value, 20).expect("failed to deserialize"); + assert_nodes(config.members, expected_nodes.clone()); + + let actual_overrides = config + .overrides + .expect("override should be found") + .into_iter() + .map(|(key, node)| (key.0, node.0)) + .collect::>(); + + assert_eq!( + actual_overrides, + expected_overrides + .into_iter() + .map(|(key, node)| (key, node.name().to_string())) + .collect::>() + ); + } +} diff --git a/osprey_coordinator/src/discovery/service.rs b/osprey_coordinator/src/discovery/service.rs new file mode 100644 index 0000000..266726f --- /dev/null +++ b/osprey_coordinator/src/discovery/service.rs @@ -0,0 +1,287 @@ +use crate::discovery::error::DiscoveryError; + +use anyhow::Result; +use log::warn; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::net::SocketAddr; +use trust_dns_resolver::TokioAsyncResolver; + +/// An advertisement of a network service. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ServiceRegistration { + pub name: String, + pub address: String, + pub port: u16, + #[serde(default)] + pub ports: HashMap, + #[serde(default)] + pub ip: Option, + #[serde(default)] + pub id_override: Option, + #[serde(default)] + pub draining: bool, +} + +impl ServiceRegistration { + /// Creates a builder for the service, allowing you to configure the service metadata and address override. + pub fn builder>(name: S, port: u16) -> Builder { + Builder { + name: name.into(), + port, + ports: HashMap::new(), + address_override: None, + ip_override: None, + id_override: None, + draining: false, + } + } +} + +impl ServiceRegistration { + pub fn id(&self) -> String { + if let Some(id_override) = self.id_override.as_ref() { + return format!("{}:{}", id_override.clone(), self.port); + } + format!("{}:{}", self.address, self.port) + } + + pub(crate) fn key(&self) -> String { + format!("/discovery/{}/instances/{}", self.name, self.id()) + } + + pub async fn socket_address(&self, resolver: TokioAsyncResolver) -> Result { + let response = resolver.lookup_ip(self.address.as_str()).await?; + let ip_addr = + response + .iter() + .next() + .ok_or_else(|| DiscoveryError::FailedToResolveHostname { + hostname: self.address.clone(), + })?; + Ok(SocketAddr::new(ip_addr, self.port)) + } +} + +pub struct Builder { + id_override: Option, + name: String, + address_override: Option, + port: u16, + ports: HashMap, + ip_override: Option, + draining: bool, +} + +impl Builder { + /// Overrides the default id that will be used in the service key. + /// + /// The default id will be the address of the service. The port is automatically + /// appended to the end; i.e. address:port. + /// + /// Do not try to override the port with this method! This will panic. + pub fn with_portless_id_override>(mut self, id_override: S) -> Self { + fn str_contains_port(input: &str) -> bool { + let segments: Vec<&str> = input.split(':').collect(); + if segments.len() > 1 { + let last_segment = segments.last().unwrap(); + return last_segment.parse::().is_ok(); + } + false + } + let id: String = id_override.into(); + assert!( + !str_contains_port(&id), + "id override must not contain a port because the port is automatically appended." + ); + self.id_override = Some(id); + self + } + + /// Overrides the default address that will be used with the service. + /// + /// The default address will be the current host's hostname, or localhost, if no hostname was able to + /// be retrieved. + pub fn with_address_override>(mut self, address_override: S) -> Self { + self.address_override = Some(address_override.into()); + self + } + + /// Sets the ip address that will be used with the service. + /// + /// By default the ip address is not passed with the registration. + pub fn with_ip_override>(mut self, ip_override: S) -> Self { + self.ip_override = Some(ip_override.into()); + self + } + + /// Overrides the ports that will be announced alongside the service. + pub fn with_ports(mut self, ports: HashMap) -> Self { + self.ports = ports; + self + } + + /// Overrides the grpc port that will be used for the service. + pub fn with_grpc_port(mut self, grpc_port: u16) -> Self { + self.ports.insert("grpc".into(), grpc_port); + self + } + + /// Overrides the http port that will be used for the service. + pub fn with_http_port(mut self, http_port: u16) -> Self { + self.ports.insert("http".into(), http_port); + self + } + + /// Overrides the drain state that will be used for the service. + /// + /// By default the drain state is `false`. + pub fn with_drain_state(mut self, draining: bool) -> Self { + self.draining = draining; + self + } + + /// Consumes the builder, creating a [`Service`]. + /// + /// ### Panics: + /// + /// Panics if the hostname as was not a utf-8 string. + pub fn build(self) -> ServiceRegistration { + ServiceRegistration { + name: self.name, + port: self.port, + ports: self.ports, + address: compute_address(self.address_override), + ip: self.ip_override, + id_override: self.id_override, + draining: self.draining, + } + } +} + +fn compute_address(address_override: Option) -> String { + if let Some(address_override) = address_override { + return address_override; + } + + if let Ok(hostname) = hostname::get() { + let hostname = hostname + .into_string() + .expect("hostname was not a valid utf-8 string."); + return append_hostname_suffix(hostname); + } + + warn!("couldn't retrieve service host name. falling back localhost."); + + "localhost".to_owned() +} + +fn append_hostname_suffix(hostname: String) -> String { + match std::env::var("DISCOVERY_HOSTNAME_SUFFIX") { + Ok(suffix) => format!("{}.{}", hostname, suffix), + Err(_) => hostname, + } +} + +#[cfg(test)] +mod test { + use super::*; + use std::collections::HashMap; + + #[test] + fn test_builder_address_override() { + let mut ports = HashMap::new(); + ports.insert("grpc".to_string(), 9001); + let service = ServiceRegistration::builder("hello", 1234) + .with_address_override("world") + .with_ports(ports) + .build(); + + assert_eq!(service.name, "hello"); + assert_eq!(service.id(), "world:1234"); + assert_eq!(service.port, 1234); + assert_eq!(*service.ports.get("grpc").unwrap(), 9001); + } + + #[test] + fn test_builder_id_override() { + let mut ports = HashMap::new(); + ports.insert("grpc".to_string(), 9001); + let service = ServiceRegistration::builder("hello", 1234) + .with_portless_id_override("id-override") + .with_ports(ports) + .build(); + + assert_eq!(service.name, "hello"); + assert_eq!(service.id(), "id-override:1234"); + assert_eq!(service.port, 1234); + assert_eq!(*service.ports.get("grpc").unwrap(), 9001); + } + + #[test] + fn test_serialization() -> Result<()> { + let service = ServiceRegistration { + name: "test_service".into(), + address: "test_address".into(), + port: 18000, + ports: HashMap::new(), + ip: Some("123.123.123".into()), + id_override: None, + draining: false, + }; + let serialized_string = serde_json::to_string(&service).unwrap(); + let deserialized_service: ServiceRegistration = + serde_json::from_str(&serialized_string).unwrap(); + assert_eq!(deserialized_service.name, "test_service"); + assert_eq!(deserialized_service.address, "test_address"); + assert_eq!(deserialized_service.port, 18000); + assert_eq!(deserialized_service.ports, HashMap::new()); + assert_eq!(deserialized_service.ip.unwrap(), "123.123.123"); + assert!(deserialized_service.id_override.is_none()); + Ok(()) + } + + #[test] + fn test_serialization_with_optional_fields() -> Result<()> { + let service = ServiceRegistration { + name: "test_service".into(), + address: "test_address".into(), + port: 18000, + ports: HashMap::new(), + ip: None, + id_override: None, + draining: false, + }; + let serialized_string = serde_json::to_string(&service).unwrap(); + let deserialized_service: ServiceRegistration = + serde_json::from_str(&serialized_string).unwrap(); + assert_eq!(deserialized_service.name, "test_service"); + assert_eq!(deserialized_service.address, "test_address"); + assert_eq!(deserialized_service.port, 18000); + assert_eq!(deserialized_service.ports, HashMap::new()); + assert!(deserialized_service.ip.is_none()); + assert!(deserialized_service.id_override.is_none()); + Ok(()) + } + + #[test] + fn test_deserialization_with_optional_fields() -> Result<()> { + let serialized_string = r#" + { + "name": "test_service", + "address": "test_address", + "port": 18000, + "ports": {} + } + "#; + let deserialized_service: ServiceRegistration = + serde_json::from_str(&serialized_string).unwrap(); + assert_eq!(deserialized_service.name, "test_service"); + assert_eq!(deserialized_service.address, "test_address"); + assert_eq!(deserialized_service.port, 18000); + assert_eq!(deserialized_service.ports, HashMap::new()); + assert!(deserialized_service.ip.is_none()); + assert!(deserialized_service.id_override.is_none()); + Ok(()) + } +} diff --git a/osprey_coordinator/src/discovery/types/mod.rs b/osprey_coordinator/src/discovery/types/mod.rs new file mode 100644 index 0000000..5154aef --- /dev/null +++ b/osprey_coordinator/src/discovery/types/mod.rs @@ -0,0 +1 @@ +pub(crate) mod ring; diff --git a/osprey_coordinator/src/discovery/types/ring.rs b/osprey_coordinator/src/discovery/types/ring.rs new file mode 100644 index 0000000..43e6424 --- /dev/null +++ b/osprey_coordinator/src/discovery/types/ring.rs @@ -0,0 +1,58 @@ +use crate::hashring::Node as HashRingNode; +use serde::Deserialize; + +#[derive(Deserialize, Debug, Default)] +pub struct RingKey(pub u64); + +#[derive(Deserialize, Debug, Clone, Default)] +pub struct RingNodeName(pub String); + +#[derive(Deserialize, Debug)] +#[serde(untagged)] +pub enum RingConfig { + LegacyMixed(Vec), + Versioned(VersionedRingConfig), +} + +#[derive(Deserialize, Eq, PartialEq, Debug)] +#[serde(untagged)] +pub enum LegacyEntry { + Named(String), + Full(HashRingNode), +} + +#[derive(Deserialize, Debug)] +#[serde(tag = "schema", rename_all = "lowercase")] +pub enum VersionedRingConfig { + V1(RingConfigV1), +} + +#[derive(Deserialize, Debug, Default)] +pub struct RingConfigV1 { + pub overrides: Option>, + pub members: Vec, +} + +impl RingConfig { + pub fn convert_to_v1(self, default_num_replicas: u32) -> RingConfigV1 { + match self { + RingConfig::LegacyMixed(members) => RingConfigV1 { + members: members + .into_iter() + .map(|entry| match entry { + LegacyEntry::Full(hash_ring_node) => hash_ring_node, + LegacyEntry::Named(name) => HashRingNode::new(name, default_num_replicas), + }) + .collect(), + overrides: None, + }, + RingConfig::Versioned(VersionedRingConfig::V1(config)) => config, + } + } +} + +impl AsRef for RingNodeName { + fn as_ref(&self) -> &str { + &self.0 + } +} diff --git a/osprey_coordinator/src/discovery/watcher.rs b/osprey_coordinator/src/discovery/watcher.rs new file mode 100644 index 0000000..5560146 --- /dev/null +++ b/osprey_coordinator/src/discovery/watcher.rs @@ -0,0 +1,291 @@ +use crate::backoff_utils::{AsyncBackoff, Config as BackoffConfig}; +use crate::discovery::error::DiscoveryError; +use crate::discovery::ring::{Config as RingConfig, Ring, DEFAULT_NUM_REPLICAS}; +use crate::discovery::service::ServiceRegistration; +use crate::etcd::{Action, Client, Node, WatchEvent, Watcher}; +use crate::tokio_utils::{AbortOnDrop, UnboundedReceiverChunker}; +use anyhow::Result; +use arc_swap::ArcSwap; +use indexmap::IndexMap; +use log::{error, info, trace}; +use rand::Rng; +use std::sync::Arc; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; +use tokio::sync::oneshot; +use tokio::task::spawn; +use tokio_stream::StreamExt; + +/// Watchers a service's instances in discovery. +#[derive(Clone, Debug)] +pub struct ServiceWatcher { + /// Key in etcd. + key: String, + /// List of instances currently in discovery. + instances: Arc>>, + /// Ring for selecting by key. + ring: Option, + /// Etcd client. + client: Client, + /// Sender that signals a stop once dropped. + _stop_tx: UnboundedSender<()>, +} + +impl ServiceWatcher { + /// Watch a service's instances. + /// + /// # Arguments + /// + /// * `name` - the service name + /// * `client` - etcd client + pub async fn watch>(name: N, client: Client, ring: bool) -> Result { + let name: String = name.into(); + let key = format!("/discovery/{}/instances", name); + let ring = if ring { + let config = RingConfig { + client: client.clone(), + default_num_replicas: DEFAULT_NUM_REPLICAS, + }; + Some(Ring::from_config(name.clone(), config).await?) + } else { + None + }; + let (_stop_tx, stop_rx) = unbounded_channel(); + let service_watcher = Self { + key, + instances: Default::default(), + client, + ring, + _stop_tx, + }; + let (ready_tx, ready_rx) = oneshot::channel(); + spawn(watch( + service_watcher.client.clone(), + service_watcher.key.clone(), + service_watcher.instances.clone(), + ready_tx, + stop_rx, + )); + ready_rx.await?; + Ok(service_watcher) + } + + /// Select a single random node from the list of instances. + /// Draining nodes are excluded. + pub fn select(&self) -> Result { + let instances_map = self.instances.load(); + let instances = instances_map + .iter() + .filter_map(|(_k, v)| (!v.draining).then_some(v)) + .collect::>(); + + let index = match instances.len() { + 0 => return Err(DiscoveryError::NoServices.into()), + 1 => 0, + len => rand::thread_rng().gen_range(0..len), + }; + + match instances.get(index) { + Some(&service) => Ok(service.clone()), + None => Err(DiscoveryError::NoServices.into()), + } + } + + /// Returns a list of all known instances. + /// Draining nodes are excluded. + pub fn select_all(&self) -> Vec { + let instances_map = self.instances.load(); + + instances_map + .iter() + .filter_map(|(_k, v)| (!v.draining).then_some(v).cloned()) + .collect::>() + } + + /// Select a number of nodes by key. + pub fn select_key, E: AsRef>( + &self, + key: K, + num_replicas: usize, + exclude: Option, + tolerate_draining: bool, + instances_to_skip: u16, + ) -> Result { + let ring = match &self.ring { + Some(ring) => ring, + None => return Err(DiscoveryError::NoRingConfigured.into()), + }; + + let nodes = match ring.select_n(key, num_replicas) { + Some(nodes) => nodes, + None => return Err(DiscoveryError::NoServices.into()), + }; + + let exclude = exclude.as_ref().map(|e| e.as_ref()); + + let mut instance_index: u16 = 0; + for node in nodes { + if let Some(exclude) = &exclude { + if &*node == *exclude { + continue; + } + } + + if let Some(service) = self.get(&node) { + if service.draining && !tolerate_draining { + continue; + } + if instance_index == instances_to_skip { + return Ok(service); + } else { + instance_index += 1; + } + } + } + Err(DiscoveryError::NoServices.into()) + } + + /// Get a Service by ID + pub fn get(&self, id: impl AsRef) -> Option { + self.instances.load().get(id.as_ref()).cloned() + } + + /// Get the number of instances + pub fn num_instances(&self) -> usize { + self.instances.load().len() + } + + /// Get the share of the ring for the given service + pub fn ring_share(&self, id: impl AsRef) -> f64 { + let ring = self + .ring + .as_ref() + .expect("Cannot calculate ring_share without a ring"); + + let mut node_replicas = 0; + let mut total_replicas = 0; + for node in &ring.nodes() { + total_replicas += node.num_replicas(); + + if node.name().eq(id.as_ref()) { + node_replicas = node.num_replicas(); + } + } + + if total_replicas > 0 { + node_replicas as f64 / total_replicas as f64 + } else { + 0.0 + } + } +} + +/// Core loop. This watches etcd for changes in the list of instances. +async fn watch( + client: Client, + key: String, + instances: Arc>>, + ready_tx: oneshot::Sender<()>, + mut stop_rx: UnboundedReceiver<()>, +) { + let mut watcher = Watcher::new(client, key); + let mut ready_tx = Some(ready_tx); + loop { + let watch_event = tokio::select! { + Some(watch_event) = watcher.next() => watch_event, + Some(_) = stop_rx.recv() => return, + else => return, + }; + + let info = match watch_event { + WatchEvent::Get(Some(info)) => info, + WatchEvent::Get(None) => { + info!("Initialized with no nodes."); + if let Some(ready_tx) = ready_tx.take() { + ready_tx.send(()).ok(); + } + continue; + } + WatchEvent::Update(info) => info, + }; + + match info.action { + Action::Get => { + if let Some(nodes) = info.node.nodes { + match deserialize_nodes(&nodes) { + Ok(services) => { + info!("Services loaded. {:?}", services); + let new_map = services + .into_iter() + .map(|service| (service.id(), service)) + .collect(); + instances.store(Arc::new(new_map)) + } + Err(e) => { + error!("Failed to deserialize nodes. {}", e); + } + } + } + } + Action::Set | Action::Create | Action::CompareAndSwap | Action::Update => { + match deserialize_node(&info.node) { + Ok(service) => { + let service_id = service.id(); + + if !instances.load().contains_key(&service_id) { + info!("Service online. {:?}", service); + } + + instances.rcu(move |inner| { + let mut map = IndexMap::clone(inner); + map.insert(service_id.clone(), service.clone()); + map + }); + } + Err(e) => { + error!("Failed to deserialize node. {}", e); + } + } + } + Action::Delete | Action::Expire | Action::CompareAndDelete => { + if let Some(node) = &info.prev_node { + match deserialize_node(node) { + Ok(service) => { + info!("Service offline. {:?}", service); + + instances.rcu(move |inner| { + let mut map = IndexMap::clone(inner); + map.remove(&service.id()); + map + }); + } + Err(e) => { + error!("Failed to deserialize node. {}", e); + } + } + } + } + } + + if let Some(ready_tx) = ready_tx.take() { + ready_tx.send(()).ok(); + } + + trace!("Services. {:?}", instances); + } +} + +/// Deserialize a list of nodes into `Service`s. +fn deserialize_nodes(nodes: &[Node]) -> Result> { + nodes + .iter() + .map(deserialize_node) + .collect::>>() +} + +/// Deserialize a node into a `Service`. +fn deserialize_node(node: &Node) -> Result { + match &node.value { + Some(value) => Ok(serde_json::from_str(value)?), + None => Err(DiscoveryError::NoNodeValue.into()), + } +} diff --git a/osprey_coordinator/src/etcd/client.rs b/osprey_coordinator/src/etcd/client.rs new file mode 100644 index 0000000..d7deb40 --- /dev/null +++ b/osprey_coordinator/src/etcd/client.rs @@ -0,0 +1,258 @@ +use crate::future_utils::time::timeout_at; +use anyhow::{Context, Result}; +use etcd::{kv, ClientBuilder, Response}; +use futures::future::TryFutureExt; +use serde::Serialize; +use std::env; +use std::time::{Duration, Instant}; + +use crate::etcd::EtcdError; + +#[derive(Debug, Clone)] +pub struct Config { + request_timeout: Duration, +} + +impl Default for Config { + fn default() -> Config { + Config { + request_timeout: Duration::from_secs(15), + } + } +} + +fn base_client_builder(key: &str) -> Result { + let endpoints_str = + env::var(key).with_context(|| format!("{} environment variable not set.", key))?; + let endpoints = endpoints_str.split(',').collect::>(); + Ok(etcd::ClientBuilder::new(&endpoints) + .with_tcp_keepalive(Duration::from_secs(30)) + .with_request_timeout(Duration::from_secs(5))) +} + +/// Http client for Etcd keys api. +#[derive(Debug, Clone)] +pub struct Client { + inner: etcd::Client, + config: Config, +} + +impl Client { + /// Create a new client from an environment variable with comma-separated addresses. + fn from_environment(key: &str) -> Result { + let builder = base_client_builder(key)?; + let config = Config::default(); + Ok(Self { + inner: builder.build(), + config, + }) + } + /// Create a new client from the ETCD_PEERS environment variable + pub fn from_etcd_peers() -> Result { + Self::from_environment("ETCD_PEERS") + } + + /// Create a new client from the ETCD_LOCK environment variable + pub fn from_etcd_lock() -> Result { + Self::from_environment("ETCD_LOCK") + } + + /// Create a new client from the ETCD_SECURE_PEERS environment variable + pub fn from_etcd_secure_peers() -> Result { + let builder = base_client_builder("ETCD_SECURE_PEERS")?; + let username = + env::var("ETCD_USERNAME").context("ETCD_USERNAME environment variable not set.")?; + let password = + env::var("ETCD_PASSWORD").context("ETCD_PASSWORD environment variable not set.")?; + let inner = builder.with_basic_auth(username, password).build(); + let config = Config::default(); + Ok(Self { inner, config }) + } + + /// Create a new client. + /// + /// # Arguments + /// + /// * `endpoints` - list of etcd endpoints. Called in order until one succeeds. + pub fn new>(endpoints: &[T]) -> Result { + let config = Config::default(); + Ok(Self { + inner: etcd::Client::new(&endpoints.iter().map(|x| x.as_ref()).collect::>()), + config, + }) + } + + /// Create a new client with config. + /// + /// # Arguments + /// + /// * `endpoints` - list of etcd endpoints. Called in order until one succeeds. + /// * `config` - configuration options + pub fn new_with_config(endpoints: &[&str], config: Config) -> Result { + Ok(Self { + inner: etcd::Client::new(endpoints), + config, + }) + } + + /// Create the value of a key. + /// Fails if the key already exists. + pub async fn create<'a, K: Into<&'a str>, O: Serialize + 'a>( + &'a self, + key: K, + object: O, + ttl: Option, + ) -> Result, EtcdError> { + let key: &str = key.into(); + let deadline = Instant::now() + self.config.request_timeout; + let value = serde_json::to_string(&object) + .map_err(|err| EtcdError::Serialize { error: err.into() })?; + let fut = kv::create(&self.inner, key, &value, ttl).map_err(EtcdError::from); + timeout_at(deadline, fut).await + } + + /// Get the value of a key. + pub async fn get<'a, K: Into<&'a str>>( + &'a self, + key: K, + ) -> Result, EtcdError> { + let key: &str = key.into(); + let deadline = Instant::now() + self.config.request_timeout; + let fut = kv::get(&self.inner, key, kv::GetOptions::default()).map_err(EtcdError::from); + timeout_at(deadline, fut).await + } + + /// Get the value of a key. + pub async fn get_recursive<'a, K: Into<&'a str>>( + &'a self, + key: K, + ) -> Result, EtcdError> { + let key: &str = key.into(); + let deadline = Instant::now() + self.config.request_timeout; + let fut = kv::get( + &self.inner, + key, + kv::GetOptions { + recursive: true, + ..Default::default() + }, + ) + .map_err(EtcdError::from); + timeout_at(deadline, fut).await + } + + /// Set the value of a key. + pub async fn set<'a, K: Into<&'a str>, O: Serialize + 'a>( + &'a self, + key: K, + object: O, + ttl: Option, + ) -> Result, EtcdError> { + let key: &str = key.into(); + let deadline = Instant::now() + self.config.request_timeout; + let value = serde_json::to_string(&object) + .map_err(|err| EtcdError::Serialize { error: err.into() })?; + let fut = kv::set(&self.inner, key, &value, ttl).map_err(EtcdError::from); + timeout_at(deadline, fut).await + } + + /// Refreshes the ttl of a key without triggering watchers. + pub async fn refresh<'a, K: Into<&'a str>>( + &'a self, + key: K, + ttl: u64, + ) -> Result, EtcdError> { + let key: &str = key.into(); + let deadline = Instant::now() + self.config.request_timeout; + let fut = kv::refresh(&self.inner, key, ttl).map_err(EtcdError::from); + timeout_at(deadline, fut).await + } + + /// Deletes a node only if the given current value and/or current modified index match. + pub async fn compare_and_delete<'a, K: Into<&'a str>>( + &'a self, + key: K, + current_value: Option<&str>, + current_modified_index: Option, + ) -> Result<(), EtcdError> { + let key: &str = key.into(); + let deadline = Instant::now() + self.config.request_timeout; + let fut = kv::compare_and_delete(&self.inner, key, current_value, current_modified_index) + .map_err(EtcdError::from); + timeout_at(deadline, fut).await.map(|_response| ()) + } + + /// Deletes a key. + pub async fn delete<'a, K: Into<&'a str>>(&'a self, key: K) -> Result<(), EtcdError> { + let key: &str = key.into(); + let deadline = Instant::now() + self.config.request_timeout; + let fut = kv::delete(&self.inner, key, false).map_err(EtcdError::from); + timeout_at(deadline, fut).await.map(|_response| ()) + } + + /// Watches a key. + /// + /// # Arguments + /// + /// * `key` - key to watch + /// * `after_index` - return events only after `after_index` + pub async fn watch_recursive<'a>( + &'a self, + key: &'a str, + after_index: u64, + ) -> Result, EtcdError> { + let watch_options = kv::WatchOptions { + index: Some(after_index + 1), + recursive: true, + timeout: None, + }; + kv::watch(&self.inner, key, watch_options) + .map_err(EtcdError::from) + .await + } + + /// Watches a key. + /// + /// # Arguments + /// + /// * `key` - key to watch + /// * `after_index` - return events only after `after_index` + pub async fn watch<'a>( + &'a self, + key: &'a str, + after_index: u64, + ) -> Result, EtcdError> { + let watch_options = kv::WatchOptions { + index: Some(after_index + 1), + recursive: false, + timeout: None, + }; + kv::watch(&self.inner, key, watch_options) + .map_err(EtcdError::from) + .await + } + + /// Watches a key, allowing various options to be provided + /// + /// # Arguments + /// * `key` - key to watch + /// * `after_index` - return events only after `after_index` + /// * `recursive` - whether the watch should be recursive + /// * `timeout` - when the watch should time out. + pub async fn watch_opt( + &self, + key: impl AsRef, + after_index: u64, + recursive: bool, + timeout: Duration, + ) -> Result, EtcdError> { + let watch_options = kv::WatchOptions { + index: Some(after_index + 1), + recursive, + timeout: Some(timeout), + }; + kv::watch(&self.inner, key, watch_options) + .map_err(EtcdError::from) + .await + } +} diff --git a/osprey_coordinator/src/etcd/error.rs b/osprey_coordinator/src/etcd/error.rs new file mode 100644 index 0000000..101e36e --- /dev/null +++ b/osprey_coordinator/src/etcd/error.rs @@ -0,0 +1,74 @@ +use etcd::kv::{self, WatchError}; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum EtcdError { + #[error("API error. index={} error={}", index, error)] + API { index: u64, error: etcd::Error }, + #[error("Key not found. index={} error={}", index, error)] + KeyNotFound { index: u64, error: etcd::Error }, + #[error("Key already exists. index={} error={}", index, error)] + KeyAlreadyExists { index: u64, error: etcd::Error }, + #[error("Watch error. error={}", error)] + Watch { error: kv::WatchError }, + #[error("Serialization error. error={}", error)] + Serialize { error: anyhow::Error }, + #[error("The response had no index.")] + NoIndex, + #[error("IoError. error={}", error)] + IoError { + #[from] + error: std::io::Error, + }, + #[error("Etcd error. error={}", error)] + Unknown { error: etcd::Error }, +} + +impl EtcdError { + pub fn index(&self) -> u64 { + match self { + EtcdError::API { index, .. } => *index, + _ => 0, + } + } +} + +impl From> for EtcdError { + fn from(errors: Vec) -> EtcdError { + let error = errors + .into_iter() + .next() + .expect("invariant: errors must not be empty"); + + match error { + etcd::Error::Api(error) => { + if error.error_code == 100 { + EtcdError::KeyNotFound { + index: error.index, + error: etcd::Error::Api(error), + } + } else if error.error_code == 105 { + EtcdError::KeyAlreadyExists { + index: error.index, + error: etcd::Error::Api(error), + } + } else { + EtcdError::API { + index: error.index, + error: etcd::Error::Api(error), + } + } + } + other => EtcdError::Unknown { error: other }, + } + } +} + +impl From for EtcdError { + fn from(error: WatchError) -> EtcdError { + match error { + WatchError::Other(errors) => EtcdError::from(errors), + error => EtcdError::Watch { error }, + } + } +} diff --git a/osprey_coordinator/src/etcd/mod.rs b/osprey_coordinator/src/etcd/mod.rs new file mode 100644 index 0000000..a32ff66 --- /dev/null +++ b/osprey_coordinator/src/etcd/mod.rs @@ -0,0 +1,13 @@ +mod client; +mod error; +mod server; +mod watcher; + +pub use client::Client; +pub use error::EtcdError; +#[cfg(feature = "test_server")] +pub use server::{Server, ServerBuilder}; +pub use watcher::{WatchEvent, Watcher}; + +pub use etcd::kv::{Action, KeyValueInfo, Node, WatchError}; +pub use etcd::Error as EtcdApiError; diff --git a/osprey_coordinator/src/etcd/server.rs b/osprey_coordinator/src/etcd/server.rs new file mode 100644 index 0000000..057e4ac --- /dev/null +++ b/osprey_coordinator/src/etcd/server.rs @@ -0,0 +1,215 @@ +#![cfg(feature = "test_server")] + +use std::convert::TryInto; +use std::ffi::OsStr; +use std::future::Future; +use std::net::{Ipv4Addr, SocketAddr}; +use std::path::Path; +use std::process::Stdio; +use std::time::Duration; + +use etcd::Client as InnerClient; +use nix::sys::signal::{self, Signal}; +use nix::unistd::Pid; +use tempfile::TempDir; +use tokio::net::TcpSocket; +use tokio::process::{Child, Command}; + +use crate::{Client, EtcdError}; + +/// Builder for [`Server`]. +#[derive(Clone, Debug)] +pub struct ServerBuilder<'a> { + exe: &'a OsStr, + data_dir: Option<&'a Path>, +} + +impl<'a> Default for ServerBuilder<'a> { + fn default() -> Self { + ServerBuilder { + exe: OsStr::new("etcd"), + data_dir: None, + } + } +} + +impl<'a> ServerBuilder<'a> { + /// Sets the etcd binary to use. + /// By default, the etcd from `PATH` will be used. + pub fn executable_path(&'a mut self, path: &'a (impl AsRef + ?Sized)) -> &'a mut Self { + self.exe = path.as_ref().as_os_str(); + self + } + + /// Sets the path to the data directory to use. + /// By default, a temporary directory will be created for the process + /// and deleted when the [`Server`] is `Drop`ped. + pub fn data_directory(&'a mut self, path: &'a (impl AsRef + ?Sized)) -> &'a mut Self { + self.data_dir = Some(path.as_ref()); + self + } + + /// Starts a new etcd process, waits for it to be ready, + /// then returns its [`Server`] handle. + pub fn start(&self) -> impl Future> + Send + 'static { + let mut command = Command::new(self.exe); + command.stdin(Stdio::null()); + command.stdout(Stdio::null()); + command.kill_on_drop(true); + + let result = (|| -> Result { + let port = pick_unused_port()?; + let peer_port = pick_unused_port()?; + let endpoint_url = format!("http://localhost:{}", port); + command.arg(format!("--listen-client-urls={}", &endpoint_url)); + command.arg(format!("--listen-peer-urls=http://localhost:{}", peer_port)); + command.arg(format!( + "--advertise-client-urls=http://localhost:{}", + peer_port + )); + let data_dir = if let Some(data_dir) = self.data_dir { + command.arg(format!("--data-dir={}", data_dir.display())); + None + } else { + let dir = tempfile::tempdir()?; + command.arg(format!("--data-dir={}", dir.path().display())); + Some(dir) + }; + Ok(Server { + endpoint_url, + child: command.spawn()?, + _data_dir: data_dir, + }) + })(); + + async move { + let mut server = result?; + + let client = InnerClient::new(&[server.endpoint_url()]); + let wait_future = server.child.wait(); + let healthy_future = wait_for_etcd_healthy(&client); + + tokio::select! { + wait_result = wait_future => { + match wait_result { + Ok(status) => Err(EtcdError::IoError { error: std::io::Error::new(std::io::ErrorKind::Other, format!("etcd exited with {}", status)) }), + Err(error) => Err(EtcdError::IoError { error }) + } + }, + _ = healthy_future => { + Ok(server) + }, + } + } + } +} + +/// A running etcd server. +/// +/// Dropping the `Server` will send a kill signal to the subprocess +/// but not wait on it to exit. +/// You should try to call [`Server::stop`] whenever possible. +#[derive(Debug)] +pub struct Server { + endpoint_url: String, + child: Child, + _data_dir: Option, // deleted on Drop +} + +impl Server { + /// Creates a new builder. + #[inline] + pub fn builder<'a>() -> ServerBuilder<'a> { + ServerBuilder::default() + } + + /// Sends an interrupt signal to the etcd process, + /// then returns a [`Future`] that resolves when the subprocess has exited. + /// + /// # Errors + /// + /// If signalling the subprocess fails, then an error will be returned. + pub fn stop(mut self) -> impl Future> + Send + 'static { + let pid = Pid::from_raw( + self.child + .id() + .expect("child already stopped") + .try_into() + .expect("invalid pid"), + ); + let kill_result = signal::kill(pid, Signal::SIGTERM); + async move { + if let Err(errno) = kill_result { + Err(std::io::Error::from(errno).into()) + } else { + self.child.wait().await.ok(); + Ok(()) + } + } + } + + /// Returns the server's etcd endpoint URL. + #[inline] + pub fn endpoint_url(&self) -> &str { + &self.endpoint_url + } + + /// Create a new client with the server as its sole endpoint. + #[inline] + pub fn new_client(&self) -> anyhow::Result { + Client::new(&[&self.endpoint_url]) + } +} + +async fn wait_for_etcd_healthy(client: &InnerClient) { + loop { + if client.health().await.iter().all(Result::is_ok) { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +fn pick_unused_port() -> std::io::Result { + let socket = TcpSocket::new_v4()?; + socket.set_reuseaddr(true)?; + socket.bind(SocketAddr::new(Ipv4Addr::UNSPECIFIED.into(), 0))?; + Ok(socket.local_addr()?.port()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn startup() { + if let Err(err) = which::which("etcd") { + eprintln!("etcd not found (skipping): {}", err); + return; + } + let server = Server::builder().start().await.unwrap(); + server.stop().await.unwrap(); + } + + #[tokio::test] + async fn basic_read() { + if let Err(err) = which::which("etcd") { + eprintln!("etcd not found (skipping): {}", err); + return; + } + let server = Server::builder().start().await.unwrap(); + // Use raw client to avoid conflating with potential client bugs. + let client = InnerClient::new(&[server.endpoint_url()]); + etcd::kv::create(&client, "/foo", "bar", None) + .await + .unwrap(); + let got = etcd::kv::get(&client, "/foo", Default::default()) + .await + .unwrap(); + assert_eq!( + got.data.node.value.as_ref().map(String::as_str), + Some("bar") + ); + server.stop().await.unwrap(); + } +} diff --git a/osprey_coordinator/src/etcd/watcher.rs b/osprey_coordinator/src/etcd/watcher.rs new file mode 100644 index 0000000..8cdac1b --- /dev/null +++ b/osprey_coordinator/src/etcd/watcher.rs @@ -0,0 +1,199 @@ +use crate::backoff_utils::{AsyncBackoff, Config}; +use crate::etcd::{Client, EtcdError}; +use etcd::{kv::KeyValueInfo, Response}; +use futures::Stream; +use log::error; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::Duration; +use tokio::time::sleep; + +type ResponseFuture = + Pin, EtcdError>> + Send>>; +type BackoffFuture = Pin + Send>>; + +/// A single event returned from watch stream. +pub enum WatchEvent { + /// The initial information when a watch is started. + /// `KeyValueInfo` is `None` if the key doesn't exist. + Get(Option), + /// A node update. + Update(KeyValueInfo), +} + +/// The next future that should be polled in the `Stream::poll_next` method. +enum PendingFuture { + Get(ResponseFuture), + Watch(ResponseFuture), + Backoff(BackoffFuture), +} + +/// A watcher that will stream watch events until it is dropped. It handles backoffs if etcd +/// goes offline or returns errors. +pub struct Watcher { + /// Etcd client. + client: Client, + /// Key to watch. + key: String, + /// Backoff for when errors happen. + backoff: AsyncBackoff, + /// The future that should be polled next during `Stream::poll_next`. + pending_future: Option, +} + +impl Watcher { + pub fn new(client: Client, key: String) -> Self { + let config = Config { + min_delay: Duration::from_secs(1), + max_delay: Duration::from_secs(15), + ..Default::default() + }; + let backoff = AsyncBackoff::new(config); + Self { + client, + key, + backoff, + pending_future: None, + } + } +} + +impl Stream for Watcher { + type Item = WatchEvent; + + fn poll_next(self: Pin<&mut Self>, context: &mut Context) -> Poll> { + let watcher = Pin::get_mut(self); + + // Order of operations. + // 1. Do a get to retrieve the initial index. + // 2. Start a watch from the index. + // 3. Loop on the watch until there is an error. + // + // If there is ever an error, start a backoff. Once the backoff is complete, go to step 1. + let (poll, next_future) = match watcher.pending_future.take() { + // Poll the current pending future if there isn't one. + Some(pending_future) => watcher.poll_pending(pending_future, context), + // There is no pending future. We are just starting the stream. Start by doing a get. + None => watcher.start_get(context), + }; + + watcher.pending_future = Some(next_future); + poll + } +} + +impl Watcher { + /// Unwrap the current `pending_future` and poll it. + fn poll_pending( + &mut self, + pending_future: PendingFuture, + context: &mut Context, + ) -> (Poll>, PendingFuture) { + match pending_future { + PendingFuture::Backoff(backoff_future) => self.poll_backoff(backoff_future, context), + PendingFuture::Watch(watch_future) => self.poll_watch(watch_future, context), + PendingFuture::Get(get_future) => self.poll_get(get_future, context), + } + } + + /// Poll the backoff future. If the backoff is ready then it returns Ready. + fn poll_backoff( + &mut self, + mut backoff_future: BackoffFuture, + context: &mut Context, + ) -> (Poll>, PendingFuture) { + match backoff_future.as_mut().poll(context) { + Poll::Ready(()) => self.start_get(context), + Poll::Pending => (Poll::Pending, PendingFuture::Backoff(backoff_future)), + } + } + + /// Poll the watch future. + fn poll_watch( + &mut self, + mut watch_future: ResponseFuture, + context: &mut Context, + ) -> (Poll>, PendingFuture) { + match watch_future.as_mut().poll(context) { + Poll::Ready(Err(error)) => self.start_backoff(context, error.into()), + Poll::Ready(Ok(response)) => { + let after_index = match response.data.node.modified_index { + Some(after_index) => after_index, + None => return self.start_backoff(context, EtcdError::NoIndex.into()), + }; + ( + Poll::Ready(Some(WatchEvent::Update(response.data))), + self.make_watch_future(after_index), + ) + } + Poll::Pending => (Poll::Pending, PendingFuture::Watch(watch_future)), + } + } + + /// Poll the get future. + fn poll_get( + &mut self, + mut get_future: ResponseFuture, + context: &mut Context, + ) -> (Poll>, PendingFuture) { + match get_future.as_mut().poll(context) { + Poll::Ready(Ok(response)) => { + let after_index = match response.cluster_info.etcd_index { + Some(after_index) => after_index, + None => return self.start_backoff(context, EtcdError::NoIndex.into()), + }; + + self.backoff.succeed(); + ( + Poll::Ready(Some(WatchEvent::Get(Some(response.data)))), + self.make_watch_future(after_index), + ) + } + Poll::Ready(Err(error)) => match error { + EtcdError::KeyNotFound { + index: after_index, + error: _error, + } => ( + Poll::Ready(Some(WatchEvent::Get(None))), + self.make_watch_future(after_index), + ), + other => self.start_backoff(context, other.into()), + }, + Poll::Pending => (Poll::Pending, PendingFuture::Get(get_future)), + } + } + + /// Start the backoff future. + fn start_backoff( + &mut self, + context: &mut Context, + error: anyhow::Error, + ) -> (Poll>, PendingFuture) { + let backoff_duration = self.backoff.fail(); + error!( + "Watch error. error={:?}. Starting backoff {:?}", + error, backoff_duration + ); + + let backoff_future = Box::pin(async move { sleep(backoff_duration).await }); + self.poll_backoff(backoff_future, context) + } + + /// Start the get future. + fn start_get(&mut self, context: &mut Context) -> (Poll>, PendingFuture) { + let client = self.client.clone(); + let key = self.key.clone(); + let get_future = Box::pin(async move { client.get(&*key).await }); + self.poll_get(get_future, context) + } + + /// Make a watch future. + fn make_watch_future(&mut self, after_index: u64) -> PendingFuture { + let client = self.client.clone(); + let key = self.key.clone(); + PendingFuture::Watch(Box::pin(async move { + client.watch_recursive(&key, after_index).await + })) + } +} diff --git a/osprey_coordinator/src/etcd_config/mod.rs b/osprey_coordinator/src/etcd_config/mod.rs new file mode 100644 index 0000000..bc6a378 --- /dev/null +++ b/osprey_coordinator/src/etcd_config/mod.rs @@ -0,0 +1,199 @@ +use std::sync::Arc; + +use crate::etcd::Client; +use crate::etcd_watcherd::{RecursiveKeyWatchEvents, Watcher}; +use anyhow::Result; + +use base64::{engine::general_purpose::STANDARD as BASE64_ENGINE, Engine}; +pub use etcd_config_derive::{Disconfig, FeatureFlags}; + +use prost::Message; + +/// Knows how to handle updates from etcd config changes. +pub trait KeyHandler { + /// Invoked anytime a configuration change to `key` has been detected. + /// `value` is the raw string contents stored at `key`. + fn handle_key_updated(&self, key: &str, value: Option<&str>); +} + +/// Validate a disconfig after the value from etcd event update has been successfully +/// decoded into a Protobuf message object. The default implementation returns the decoded protobuf object. +/// The type placehoder `T` is the Protobuf message object that the disconfig will be decoded into. +/// The `validate` method can be overridden to perform customized validation on the decoded Protobuf object. +pub trait HandleDisconfigUpdated { + type Disconfig: Message + Default; + type Error; + + /// If customizing the validation, return `None` if the decoded Protobuf object is invalid. + fn validate(proto: Self::Disconfig) -> Result { + Ok(proto) + } + + /// Invoked after the in-memory ArcSwap wrapped Disconfig has been updated. + fn after_update(&self) {} +} + +/// Creates a (non-secure) Etcd client to watch for any value changes recursively under the `config_key_root` path. +/// This watcher is kept alive in the background. +/// Updates to the values under the `config_key_root` are sent to the [KeyHandler] to handle. +pub async fn run_etcd_watcher( + config_key_root: impl Into, + key_handler: Arc, +) -> Result<()> { + let etcd_client = Client::from_etcd_peers()?; + // let etcd_client = Arc::new(etcd_client); + let watcher = Watcher::new(etcd_client); + let response = watcher.watch_key_recursive(config_key_root).await?; + + let mut events = response.events(); + // Handle full sync event before resolving the future. + if let RecursiveKeyWatchEvents::FullSync { items } = events.initial_event() { + for (key, value) in items.iter() { + key_handler.handle_key_updated(key, Some(value)); + } + } + + // Create background task to monitor for changes + tokio::spawn(async move { + loop { + let event = events.next().await; + match event { + RecursiveKeyWatchEvents::FullSync { items } => { + for (key, value) in items.iter() { + key_handler.handle_key_updated(key, Some(value)); + } + } + RecursiveKeyWatchEvents::SyncOne { key, value } => { + key_handler.handle_key_updated(&key, Some(&value)); + } + RecursiveKeyWatchEvents::DeleteOne { key, prev_value: _ } => { + key_handler.handle_key_updated(&key, None); + } + } + } + }); + + Ok(()) +} + +const PATTERN: &[char] = &['"', ' ']; + +pub fn convert_to_bool(value: Option<&str>) -> bool { + value.map_or(false, |v| { + v.trim_matches(PATTERN).parse::().unwrap_or_default() > 0.0 + }) +} + +pub fn convert_to_float(value: Option<&str>) -> f32 { + value.map_or(0.0, |v| { + v.trim_matches(PATTERN).parse::().unwrap_or_default() + }) +} + +pub fn convert_to_usize(value: Option<&str>) -> usize { + value.map_or(0, |v| { + v.trim_matches(PATTERN).parse::().unwrap_or_default() + }) +} + +pub fn convert_to_u64(value: Option<&str>) -> u64 { + value.map_or(0, |v| { + v.trim_matches(PATTERN).parse::().unwrap_or_default() + }) +} + +pub fn convert_to_u32(value: Option<&str>) -> u32 { + value.map_or(0, |v| { + v.trim_matches(PATTERN).parse::().unwrap_or_default() + }) +} + +pub fn base64_to_proto(value: Option<&str>) -> Result> { + let parsed_str = match value { + Some(value) => match BASE64_ENGINE.decode(value.as_bytes()) { + Ok(binary) => binary, + Err(e) => return Err(anyhow::anyhow!("Failed to decode base64 value: {:?}", e)), + }, + None => return Ok(None), + }; + + let proto = T::decode(parsed_str.as_slice()) + .map_err(|e| anyhow::anyhow!("Failed to decode protobuf: {:?}", e))?; + Ok(Some(proto)) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_convert_to_float() { + assert_eq!(0.0, convert_to_float(None)); + assert_eq!(0.0, convert_to_float(Some(""))); + assert_eq!(0.0, convert_to_float(Some(" "))); + + assert_eq!(0.0, convert_to_float(Some("0"))); + assert_eq!(0.0, convert_to_float(Some("0.0"))); + assert_eq!(0.0, convert_to_float(Some(" 0.0 "))); + assert_eq!(0.1234, convert_to_float(Some("0.1234"))); + assert_eq!(0.1234, convert_to_float(Some("\" 0.1234 \""))); + + assert_eq!(1.0, convert_to_float(Some("1.0"))); + assert_eq!(1.0, convert_to_float(Some(" 1.0 "))); + } + + #[test] + fn test_convert_to_bool() { + assert!(!convert_to_bool(None)); + assert!(!convert_to_bool(Some(""))); + assert!(!convert_to_bool(Some(" "))); + + assert!(!convert_to_bool(Some("0"))); + assert!(!convert_to_bool(Some("0.0"))); + assert!(!convert_to_bool(Some(" 0.0 "))); + assert!(convert_to_bool(Some("0.1234"))); + assert!(convert_to_bool(Some("\" 0.1234 \""))); + + assert!(convert_to_bool(Some("1.0"))); + assert!(convert_to_bool(Some(" 1.0 "))); + } + + #[test] + fn test_convert_to_usize() { + assert_eq!(0, convert_to_usize(None)); + assert_eq!(0, convert_to_usize(Some(""))); + assert_eq!(0, convert_to_usize(Some(" "))); + + assert_eq!(0, convert_to_usize(Some("0"))); + assert_eq!(0, convert_to_usize(Some("0.0"))); + assert_eq!(0, convert_to_usize(Some(" 0.0 "))); + assert_eq!(0, convert_to_usize(Some("1.0"))); + assert_eq!(0, convert_to_usize(Some("-1"))); + assert_eq!(1234, convert_to_usize(Some("1234"))); + assert_eq!(1234, convert_to_usize(Some("\" 1234 \""))); + + assert_eq!(1, convert_to_usize(Some("1"))); + assert_eq!(1, convert_to_usize(Some(" 1 "))); + } + + #[test] + fn test_convert_to_u64() { + assert_eq!(0, convert_to_u64(None)); + assert_eq!(0, convert_to_u64(Some(""))); + assert_eq!(0, convert_to_u64(Some(" "))); + + assert_eq!(0, convert_to_u64(Some("0"))); + assert_eq!(0, convert_to_u64(Some("0.0"))); + assert_eq!(0, convert_to_u64(Some(" 0.0 "))); + assert_eq!(0, convert_to_u64(Some("1.0"))); + assert_eq!(0, convert_to_u64(Some("-1"))); + assert_eq!(1234, convert_to_u64(Some("1234"))); + assert_eq!(1234, convert_to_u64(Some("\" 1234 \""))); + assert_eq!(1234, convert_to_u64(Some("1234"))); + assert_eq!(1234, convert_to_u64(Some("1234"))); + assert_eq!( + 1227039953469571133, + convert_to_u64(Some("1227039953469571133")) + ); + } +} diff --git a/osprey_coordinator/src/etcd_watcherd/api.rs b/osprey_coordinator/src/etcd_watcherd/api.rs new file mode 100644 index 0000000..a6b7793 --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/api.rs @@ -0,0 +1,292 @@ +use std::{fmt::Debug, sync::Arc, time::Duration}; + +use crate::cached_futures::CachedFutures; +use crate::metrics::counters::StaticCounter; +use crate::tokio_utils::AbortOnDrop; +use anyhow::Result; +use parking_lot::RwLock; +use tokio::{sync::mpsc, task::JoinHandle, time::interval}; + +use crate::etcd_watcherd::{ + KeyWatchEvents, RecursiveKeyWatchEvents, WatchEvents, WatcherInner, WatcherLoop, WatcherStats, + WatcherSubscription, WatcherSubscriptionId, +}; + +const WATCHER_SWEEP_INTERVAL: Duration = Duration::from_secs(120); + +#[derive(Clone, Debug, Default)] +pub(crate) struct ActiveWatchers { + pub(crate) key: CachedFutures, Arc>, + pub(crate) recursive_key: + CachedFutures, Arc>, +} + +#[derive(Clone, Debug)] +pub struct Watcher { + client: crate::etcd::Client, + _sweeper_abort_on_drop: Arc>, + pub(crate) active_watchers: ActiveWatchers, + pub(crate) stats: Arc, +} + +impl Watcher { + /// Creates a new [`Watcher`], using the given etcd client. + /// + /// Ideally for the lifespan of your application, you should use a single watcher instance. + /// + /// This watcher instance will multiplex all watches to etcd, so it makes senes to use it as a + /// singleton within your application. + pub fn new(client: crate::etcd::Client) -> Self { + let active_watchers = ActiveWatchers::default(); + let stats = Arc::new(WatcherStats::default()); + let sweeper_abort_on_drop = Arc::new(AbortOnDrop::new(tokio::task::spawn( + Self::sweep_watchers(active_watchers.clone(), stats.clone()), + ))); + + Self { + client, + active_watchers, + _sweeper_abort_on_drop: sweeper_abort_on_drop, + stats, + } + } + + /// This background task is responsible for cleaning up watchers that no longer are being actively used. + async fn sweep_watchers(active_watchers: ActiveWatchers, stats: Arc) { + let mut interval = interval(WATCHER_SWEEP_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + interval.tick().await; + + let mut watchers_swept = 0; + watchers_swept += active_watchers + .key + .retain_ready(|handle| handle.num_handles() > 1); + watchers_swept += active_watchers + .recursive_key + .retain_ready(|handle| handle.num_handles() > 1); + + stats.watchers_swept.incr_by(watchers_swept as _); + } + } + + /// Watches a single key for changes. + /// + /// Returns a [`WatchHandle`] that can be used to acquire the current value of the key at any given time, + /// or subscribe to real time updates to that key. + pub async fn watch_key(&self, key: impl Into) -> Result> { + let key = key.into(); + let client = self.client.clone(); + let stats = self.stats.clone(); + let watcher_future = + self.active_watchers + .key + .get_or_cache_default(key, move |key| async move { + use crate::etcd_watcherd::watcher_impl::key_watcher::KeyWatcher; + let watcher_impl = KeyWatcher::new(client, key); + let handle = WatcherLoop::start(watcher_impl, stats) + .await + .map_err(Arc::new)?; + Ok(handle) + }); + + watcher_future + .await + .map_err(|e| anyhow::anyhow!("etcd error: {:?}", e)) + } + + /// Watches the keys within a directory for changes. + /// + /// Returns a [`WatchHandle`] that can be used to acquire the current value of the keys within the directory + /// at any given time, or subscribe to real time updates to that key. + pub async fn watch_key_recursive( + &self, + key: impl Into, + ) -> Result> { + let key = key.into(); + let client = self.client.clone(); + let stats = self.stats.clone(); + let watcher_future = self.active_watchers.recursive_key.get_or_cache_default( + key, + move |key| async move { + use crate::etcd_watcherd::watcher_impl::recursive_key_watcher::RecursiveKeyWatcher; + let watcher_impl = RecursiveKeyWatcher::new(client, key); + let handle = WatcherLoop::start(watcher_impl, stats) + .await + .map_err(Arc::new)?; + Ok(handle) + }, + ); + + watcher_future + .await + .map_err(|e| anyhow::anyhow!("etcd error: {:?}", e)) + } +} + +/// Returned by the [`Watcher::watch_key`] or [`Watcher::watch_key_recursive`] methods. +/// +/// Allows you to read the key's current value with [`WatchHandle::current_value`], and also subscribe to updates +/// to the key using [`WatchHandle::events`]. +#[derive(Clone)] +pub struct WatchHandle +where + E: WatchEvents, +{ + /// Holds an arc to the WatcherInner that contains the state of the watched key, and the + /// means to establish additional subscriptions. + inner: Arc>>, + /// A join handle to the background task that is keeping the watcher synced with etcd. + watcher_loop_abort_on_drop: Arc>, +} + +impl WatchHandle +where + E: WatchEvents, +{ + pub(crate) fn new( + inner: Arc>>, + watcher_loop_join_handle: JoinHandle<()>, + ) -> Self { + Self { + inner, + watcher_loop_abort_on_drop: Arc::new(AbortOnDrop::new(watcher_loop_join_handle)), + } + } + + /// Returns the current value of the key. This is a snapshotted value, meaning that it is immutable. + pub fn current_value(&self) -> E::Value { + self.inner.read().current_value().clone() + } + + /// Consumes the [`WatchHandle`] and returns a [`WatcherEvents`], which allows your application to be notified of changes to watched keys in real time. + /// + /// We will try and emit the minimal set of events and eliminate duplicate events that would net in no effective state change. What this means is + /// that if a key is set to the same value multiple times in etcd, where as etcd might update the watcher internally, we will not emit these events + /// as they result in no effective change of state. + /// + /// # Example: + /// ```no_run + /// # use anyhow::Result; + /// # use etcd_watcherd::{Watcher, KeyWatchEvents}; + /// # async fn example() -> Result<()> { + /// let watcher = Watcher::new(crate::etcd::Client::from_etcd_peers()?); + /// let mut events = watcher.watch_key("/hello").await?.events(); + /// loop { + /// let KeyWatchEvents::FullSync { value } = events.next().await; + /// println!("key updated: {:?}", value); + /// } + /// # Ok(()) + /// # } + /// ``` + pub fn events(self) -> WatcherEvents { + let WatcherSubscription { + subscription_id, + receiver, + full_sync_event, + } = self.inner.write().create_subscription(); + + WatcherEvents { + handle: self, + subscription_id, + initial_event: Some(Box::new(full_sync_event)), + receiver, + } + } + + /// Returns the number of active subscribers for this watch handle. + pub(crate) fn num_subscribers(&self) -> usize { + self.inner.read().num_subscribers() + } + + /// Returns the number of copies of this watch handle that are currently active in the + /// program. + fn num_handles(&self) -> usize { + // We are using the strong count of the loop's abort on drop, since this will correspond 1:1 + // with the number of copies this handle has active. + Arc::strong_count(&self.watcher_loop_abort_on_drop) + } +} + +/// Returned by [`WatchHandle::events`]. +/// +/// Acts as a stream of events that can be consumed to be kept up-to-date on the changes to an etcd key +/// in real time. +pub struct WatcherEvents +where + E: WatchEvents, +{ + handle: WatchHandle, + subscription_id: WatcherSubscriptionId, + receiver: mpsc::Receiver, + // a box is chosen here to reduce the size of the struct, as this will be empty most of the time, + // as this event is consumed by the first call to [`WatcherEvents::next`], or [`WatcherEvents::initial_event`]. + initial_event: Option>, +} + +impl Drop for WatcherEvents +where + E: WatchEvents, +{ + fn drop(&mut self) { + let mut inner = self.handle.inner.write(); + inner.remove_subscription(self.subscription_id); + } +} + +impl WatcherEvents +where + E: WatchEvents, +{ + /// Returns the "initial watch event", which is either [`crate::RecursiveKeyWatchEvents::FullSync`] or [`crate::KeyWatchEvents::FullSync`] + /// depending on what kind of watcher you have initialized (recursive or not). + /// + /// # Panics + /// This method must be called before the first call to `next`, and must only be called once. If these invariants are not upheld, this method + /// will cause a panic. + pub fn initial_event(&mut self) -> E { + *self.initial_event.take().expect("invariant: `initial_event` can only be called once, and before the first call to `next`.") + } + + /// Returns a new copy of the [`WatchHandle`] that this [`WatcherEvents`] is using. + pub fn handle(&self) -> WatchHandle { + self.handle.clone() + } + + /// Awaits a the next key change event. + /// + /// If `initial_event` was not called, this method will return immediately the first time called with the initial event, otherwise + /// will wait until an event is received. + /// + /// It is *safe* to drop this future and re-call it, (for example, wrapping this with a `timeout`, or using the `select!` macro.) + /// No events will be dropped if this future is cancelled, as long as the return value is not dropped. + pub async fn next(&mut self) -> E { + if let Some(event) = self.initial_event.take() { + return *event; + } + + match self.receiver.recv().await { + Some(event) => event, + // The subscription has ended, which means we've fallen too far behind,so we'll automatically re-create the subscription, + // and emit a full sync event. + None => self.recreate_subscription(), + } + } + + fn recreate_subscription(&mut self) -> E { + let WatcherSubscription { + subscription_id, + receiver, + full_sync_event, + } = self + .handle + .inner + .write() + .recreate_subscription(self.subscription_id); + + self.receiver = receiver; + self.subscription_id = subscription_id; + full_sync_event + } +} diff --git a/osprey_coordinator/src/etcd_watcherd/bin/etcd_watcherd_cli.rs b/osprey_coordinator/src/etcd_watcherd/bin/etcd_watcherd_cli.rs new file mode 100644 index 0000000..a5248e4 --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/bin/etcd_watcherd_cli.rs @@ -0,0 +1,75 @@ +use crate::etcd_watcherd::{KeyWatchEvents, RecursiveKeyWatchEvents, Watcher}; +use anyhow::Result; +use clap::{Parser, Subcommand}; + +#[derive(Subcommand)] +enum Command { + /// Watches a single key for changes, using [`Watcher::watch_key`] + WatchKey { + #[arg(long)] + key: String, + }, + /// Recursively watches a directory for changes, using [`Watcher::watch_key_recursive`]. + WatchKeyRecursive { + #[arg(long)] + key: String, + }, +} + +/// A cli that uses the Watcher provided by etcd_watcherd. Different than `rpc_client`, +/// which connects to an already running server. +/// +/// Prints out events from watched keys in real time. This cli should really not be used, +/// and more or less serves as an example of using the [`Watcher`] API. +#[derive(Parser)] +struct Opt { + /// The etcd peers to connect to. If not provided, will look in the `ETCD_PEERS` environment + /// variable. + #[arg(long)] + etcd_peers: Vec, + + #[command(subcommand)] + command: Command, +} + +#[tokio::main] +async fn main() -> Result<()> { + let opt = Opt::parse(); + let etcd_client = if opt.etcd_peers.is_empty() { + crate::etcd::Client::from_etcd_peers()? + } else { + crate::etcd::Client::new(&opt.etcd_peers)? + }; + let watcher = Watcher::new(etcd_client); + + match opt.command { + Command::WatchKey { key } => { + let handle = watcher.watch_key(key.clone()).await?; + + let mut events = handle.events(); + loop { + let KeyWatchEvents::FullSync { value } = events.next().await; + println!("{:?} -> {:?}", key, value); + } + } + Command::WatchKeyRecursive { key } => { + let response = watcher.watch_key_recursive(key.clone()).await?; + let mut events = response.events(); + + loop { + let event = events.next().await; + match event { + RecursiveKeyWatchEvents::FullSync { items } => { + println!("full-sync: {:#?}", items) + } + RecursiveKeyWatchEvents::SyncOne { key, value } => { + println!("sync-one: {:?} -> {:?}", key, value) + } + RecursiveKeyWatchEvents::DeleteOne { key, prev_value } => { + println!("delete-one: {:?} (prev: {:?})", key, prev_value) + } + } + } + } + } +} diff --git a/osprey_coordinator/src/etcd_watcherd/bin/etcd_watcherd_rpc_client.rs b/osprey_coordinator/src/etcd_watcherd/bin/etcd_watcherd_rpc_client.rs new file mode 100644 index 0000000..56c3663 --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/bin/etcd_watcherd_rpc_client.rs @@ -0,0 +1,72 @@ +use anyhow::Result; + +use crate::etcd_watcherd::rpc::client::*; +use clap::{Parser, Subcommand}; +use watch_key_recursive_response::Event; + +#[derive(Subcommand)] +enum Command { + /// Watches a single key for changes. + WatchKey { + #[arg(long)] + key: String, + }, + /// Recursively watches a directory for changes. + WatchKeyRecursive { + #[arg(long)] + key: String, + }, +} + +/// A cli client for `etcd_watcherd`. +/// +/// Prints out events from watched keys in real time. +#[derive(Parser)] +struct Opt { + /// The address of the watcherd server to connect to. + #[arg(long, default_value = "http://localhost:2375")] + watcherd_server: String, + + #[command(subcommand)] + command: Command, +} + +#[tokio::main] +async fn main() -> Result<()> { + let opt = Opt::parse(); + let mut client = EtcdWatcherdServiceClient::connect(opt.watcherd_server).await?; + match opt.command { + Command::WatchKey { key } => { + let response = client + .watch_key(WatchKeyRequest { key: key.clone() }) + .await?; + + let mut response = response.into_inner(); + while let Some(message) = response.message().await? { + println!("{:?} -> {:?}", key, message.value); + } + } + Command::WatchKeyRecursive { key } => { + let response = client + .watch_key_recursive(WatchKeyRecursiveRequest { key }) + .await?; + + let mut response = response.into_inner(); + while let Some(message) = response.message().await? { + match message.event.unwrap() { + Event::FullSync(s) => { + println!("full-sync: {:#?}", s.items) + } + Event::SyncOne(s) => { + println!("sync-one: {:?} -> {:?}", s.key, s.value) + } + Event::DeleteOne(s) => { + println!("delete-one: {:?} (prev: {:?})", s.key, s.prev_value) + } + } + } + } + } + + Ok(()) +} diff --git a/osprey_coordinator/src/etcd_watcherd/bin/etcd_watcherd_rpc_server.rs b/osprey_coordinator/src/etcd_watcherd/bin/etcd_watcherd_rpc_server.rs new file mode 100644 index 0000000..eeb1104 --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/bin/etcd_watcherd_rpc_server.rs @@ -0,0 +1,52 @@ +use std::{net::SocketAddr, time::Duration}; + +use crate::etcd::Client; +use crate::etcd_watcherd::rpc::server::{EtcdWatcherdServiceImpl, EtcdWatcherdServiceServer}; +use crate::etcd_watcherd::Watcher; +use anyhow::Result; +use clap::Parser; +use tonic::transport::Server; + +/// `etcd_watcherd` is an etcd watch multiplexer, establishing a single watcher to etcd for each key +/// requested to be watched by the clients. +/// +/// This server is meant to be deployed on the same nodes those Python processes run on, either as a side-car, +/// or a daemon-set. +/// +/// The watcher will listen on the provided port, and multiplex connections to the etcd cluster whose peers +/// are specified in the environmental variable `ETCD_PEERS`. +#[derive(Parser)] +struct Opt { + /// The address that the `etcd_watcherd` server should bind to. + #[arg(long, default_value = "127.0.0.1:2375")] + bind_addr: SocketAddr, + + /// If provided, overrides the peers specified by `ETCD_PEERS`. + #[arg(long)] + etcd_peers: Vec, +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<()> { + let opt = Opt::parse(); + let client = if opt.etcd_peers.is_empty() { + Client::from_etcd_peers()? + } else { + Client::new(&opt.etcd_peers)? + }; + + let metrics = metrics::new_client("etcd_watcherd.rpc_server")?; + let watcher = Watcher::new(client); + let _metrics_guard = watcher.emit_metrics(metrics); + let watcherd_impl = EtcdWatcherdServiceImpl::new(watcher); + + println!("[etcd_watcherd] serving on {:?}", opt.bind_addr); + + Server::builder() + .tcp_keepalive(Some(Duration::from_secs(10))) + .add_service(EtcdWatcherdServiceServer::new(watcherd_impl)) + .serve(opt.bind_addr) + .await?; + + Ok(()) +} diff --git a/osprey_coordinator/src/etcd_watcherd/metrics.rs b/osprey_coordinator/src/etcd_watcherd/metrics.rs new file mode 100644 index 0000000..81a98cc --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/metrics.rs @@ -0,0 +1,118 @@ +use std::time::Duration; + +use crate::cached_futures::{CachedFutureRef, CachedFutures}; +use crate::metrics::{EmitMetrics, SharedMetrics}; +use tokio::{ + task::JoinHandle, + time::{interval, MissedTickBehavior}, +}; + +use crate::etcd_watcherd::{watcher_impl::WatchEvents, WatchHandle, Watcher}; + +pub struct WatcherMetrics { + watcher: Watcher, + metrics: SharedMetrics, +} + +impl Watcher { + /// Spawns a worker that will emit metrics to the provided [`SharedMetrics`]. + /// + /// This worker will last for as long as the returned [`WatcherMetricsGuard`] lasts, + /// and will automatically shut down when it is dropped. + pub fn emit_metrics(&self, metrics: SharedMetrics) -> WatcherMetricsGuard { + let watcher = self.clone(); + let metrics = WatcherMetrics { watcher, metrics }; + let emit_loop_join_handle = tokio::task::spawn(metrics.emit_loop()); + + WatcherMetricsGuard { + emit_loop_join_handle, + } + } +} + +impl WatcherMetrics { + async fn emit_loop(mut self) { + let mut interval = interval(Duration::from_secs(10)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + loop { + interval.tick().await; + self.emit_metrics().await; + } + } + + async fn emit_metrics(&mut self) { + emit_watcher_counts("key", &self.metrics, &self.watcher.active_watchers.key); + emit_watcher_counts( + "recursive_key", + &self.metrics, + &self.watcher.active_watchers.recursive_key, + ); + + emit_subscriber_counts("key", &self.metrics, &self.watcher.active_watchers.key); + emit_subscriber_counts( + "recursive_key", + &self.metrics, + &self.watcher.active_watchers.recursive_key, + ); + + self.watcher.stats.emit_metrics(&self.metrics).await; + } +} + +pub struct WatcherMetricsGuard { + emit_loop_join_handle: JoinHandle<()>, +} + +impl Drop for WatcherMetricsGuard { + fn drop(&mut self) { + self.emit_loop_join_handle.abort(); + } +} + +fn emit_watcher_counts( + watcher_type: &str, + metrics: &SharedMetrics, + cached_futures: &CachedFutures, +) where + K: Eq + std::hash::Hash + Clone + Send + Sync + 'static, + T: Clone + Send + Sync + 'static, + E: Clone + Send + Sync + 'static, +{ + let (active_watchers, pending_watchers) = + cached_futures.fold((0, 0), |(active, pending), (_, fut_ref)| match fut_ref { + CachedFutureRef::Ready(_) => (active + 1, pending), + CachedFutureRef::Pending => (active, pending + 1), + }); + + metrics + .gauge_with_tags("watchers", active_watchers) + .with_tag("status", "active") + .with_tag("type", watcher_type) + .send(); + + metrics + .gauge_with_tags("watchers", pending_watchers) + .with_tag("status", "pending") + .with_tag("type", watcher_type) + .send(); +} + +fn emit_subscriber_counts( + watcher_type: &str, + metrics: &SharedMetrics, + cached_futures: &CachedFutures, E>, +) where + K: Eq + std::hash::Hash + Clone + Send + Sync + 'static, + T: Clone + Send + Sync + WatchEvents + 'static, + E: Clone + Send + Sync + 'static, +{ + let subscribers = cached_futures.fold_ready(0, |subscribers, (_, handle)| { + subscribers + handle.num_subscribers() + }); + + metrics + .gauge_with_tags("subscribers", subscribers as u64) + .with_tag("type", watcher_type) + .send(); +} diff --git a/osprey_coordinator/src/etcd_watcherd/mod.rs b/osprey_coordinator/src/etcd_watcherd/mod.rs new file mode 100644 index 0000000..873e001 --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/mod.rs @@ -0,0 +1,20 @@ +mod api; +mod metrics; +mod watcher; +mod watcher_impl; + +#[cfg(feature = "rpc")] +pub mod rpc; +mod stats; + +pub use self::api::{WatchHandle, Watcher, WatcherEvents}; +pub use self::metrics::{WatcherMetrics, WatcherMetricsGuard}; +pub use self::watcher_impl::{ + key_watcher::KeyWatchEvents, recursive_key_watcher::RecursiveKeyWatchEvents, +}; + +pub(crate) use self::stats::WatcherStats; +pub(crate) use self::watcher::{ + WatcherInner, WatcherLoop, WatcherSubscription, WatcherSubscriptionId, +}; +pub(crate) use self::watcher_impl::{WatchEvents, WatchResult, WatchState, WatcherImpl}; diff --git a/osprey_coordinator/src/etcd_watcherd/rpc/client.rs b/osprey_coordinator/src/etcd_watcherd/rpc/client.rs new file mode 100644 index 0000000..c974d73 --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/rpc/client.rs @@ -0,0 +1,4 @@ +pub use super::proto::v1::{ + etcd_watcherd_service_client::EtcdWatcherdServiceClient, watch_key_recursive_response, + WatchKeyRecursiveRequest, WatchKeyRecursiveResponse, WatchKeyRequest, WatchKeyResponse, +}; diff --git a/osprey_coordinator/src/etcd_watcherd/rpc/mod.rs b/osprey_coordinator/src/etcd_watcherd/rpc/mod.rs new file mode 100644 index 0000000..ec0a014 --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/rpc/mod.rs @@ -0,0 +1,8 @@ +mod proto { + pub mod v1 { + tonic::include_proto!("discord_common.etcd_watcherd.v1"); + } +} + +pub mod client; +pub mod server; diff --git a/osprey_coordinator/src/etcd_watcherd/rpc/server.rs b/osprey_coordinator/src/etcd_watcherd/rpc/server.rs new file mode 100644 index 0000000..c6f1205 --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/rpc/server.rs @@ -0,0 +1,118 @@ +use std::pin::Pin; + +use futures::Stream; +use tonic::{Response, Status}; + +use crate::etcd_watcherd::{ + rpc::proto::v1 as proto, KeyWatchEvents, RecursiveKeyWatchEvents, Watcher, +}; +pub use proto::etcd_watcherd_service_server::EtcdWatcherdServiceServer; + +pub struct EtcdWatcherdServiceImpl { + watcher_client: Watcher, +} + +impl EtcdWatcherdServiceImpl { + pub fn new(watcher_client: Watcher) -> Self { + EtcdWatcherdServiceImpl { watcher_client } + } +} + +type BoxedStream = Pin> + Send + Sync + 'static>>; + +#[tonic::async_trait] +impl proto::etcd_watcherd_service_server::EtcdWatcherdService for EtcdWatcherdServiceImpl { + type WatchKeyStream = BoxedStream; + + async fn watch_key( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let request = request.into_inner(); + if !request.key.starts_with("/") { + return Err(tonic::Status::invalid_argument( + "`key` must start with a '/'", + )); + } + if request.key.ends_with("/") { + return Err(tonic::Status::invalid_argument( + "`key` must not end with a '/'", + )); + } + + let watch_handle = self + .watcher_client + .watch_key(request.key) + .await + .map_err(|e| Status::internal(format!("etcd watcher error: {:?}", e)))?; + + let mut events = watch_handle.events(); + let output = async_stream::try_stream! { + loop { + let KeyWatchEvents::FullSync { value } = events.next().await; + let value = value.map(|x| x.to_string()); + let response = proto::WatchKeyResponse { value }; + yield response; + } + }; + Ok(Response::new(Box::pin(output) as Self::WatchKeyStream)) + } + + type WatchKeyRecursiveStream = BoxedStream; + + async fn watch_key_recursive( + &self, + request: tonic::Request, + ) -> Result, Status> { + let request = request.into_inner(); + if !request.key.starts_with("/") { + return Err(tonic::Status::invalid_argument( + "`key` must start with a '/'", + )); + } + if !request.key.ends_with("/") { + return Err(tonic::Status::invalid_argument("`key` must end with a '/'")); + } + + let watch_handle = self + .watcher_client + .watch_key_recursive(request.key) + .await + .map_err(|e| Status::internal(format!("etcd watcher error: {:?}", e)))?; + + let mut events = watch_handle.events(); + let output = async_stream::try_stream! { + use proto::watch_key_recursive_response::*; + loop { + let event = match events.next().await { + RecursiveKeyWatchEvents::FullSync { items } => { + let items = items + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + Event::FullSync(FullSync { items }) + } + RecursiveKeyWatchEvents::SyncOne { key, value } => { + let (key, value) = (key.to_string(), value.to_string()); + + Event::SyncOne(SyncOne { key, value }) + } + RecursiveKeyWatchEvents::DeleteOne { key, prev_value } => { + let (key, prev_value) = (key.to_string(), prev_value.to_string()); + + Event::DeleteOne(DeleteOne { key, prev_value }) + } + }; + let response = proto::WatchKeyRecursiveResponse { + event: Some(event) + }; + yield response; + } + }; + + Ok(Response::new( + Box::pin(output) as Self::WatchKeyRecursiveStream + )) + } +} diff --git a/osprey_coordinator/src/etcd_watcherd/stats.rs b/osprey_coordinator/src/etcd_watcherd/stats.rs new file mode 100644 index 0000000..f86592c --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/stats.rs @@ -0,0 +1,28 @@ +use crate::metrics::define_metrics; + +define_metrics!(WatcherStats, [ + // Incremented when an etcd error occurs during the watcher loop. + watch_etcd_errors => StaticCounter(), + // Incremented when a watch fails to start. + watcher_start_fails => StaticCounter(), + // Incremented when a watcher successfully starts. + watcher_start_success => StaticCounter(), + // Incremented when a watcher has been swept. + watchers_swept => StaticCounter(), + // Incremented when an etcd watch request returns a result. + watch_success => StaticCounter(), + // Incremented when a watch times out. + watch_timeout => StaticCounter(), + // Incremented when a watch resets. + watch_reset => StaticCounter(), + // Incremented when a new subscription is created. + subscriptions_created => StaticCounter(), + // Incremented when a subscription is removed. + subscriptions_removed => StaticCounter(), + // Incremented when a subscription has been re-created. + subscriptions_recreated => StaticCounter(), + // Incremented when a subscription falls too far behind and gets reset. + subscriptions_reset => StaticCounter(), + // Incremented when a subscription has been notified of an event. + subscriptions_notified => StaticCounter(), +]); diff --git a/osprey_coordinator/src/etcd_watcherd/watcher.rs b/osprey_coordinator/src/etcd_watcherd/watcher.rs new file mode 100644 index 0000000..3d2c9db --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/watcher.rs @@ -0,0 +1,276 @@ +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use crate::backoff_utils::{AsyncBackoff, Config as AsyncBackoffConfig}; +use crate::metrics::counters::StaticCounter; +use parking_lot::RwLock; +use tokio::{sync::mpsc, time::sleep}; +use tracing::{error, info}; + +use crate::etcd_watcherd::{ + WatchEvents, WatchHandle, WatchResult, WatchState, WatcherImpl, WatcherStats, +}; + +pub(crate) struct WatcherInner +where + E: WatchEvents, +{ + #[allow(dead_code)] + key: String, + current_value: E::Value, + next_subscription_id: usize, + watchers: HashMap>, + stats: Arc, +} + +pub(crate) struct WatcherSubscription +where + E: WatchEvents, +{ + pub subscription_id: WatcherSubscriptionId, + pub receiver: mpsc::Receiver, + pub full_sync_event: E, +} + +/// A subscription id is created by [`WatcherInner::create_subscription`] and is used to identify +/// a given subscription within the system. +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +pub(crate) struct WatcherSubscriptionId(usize); + +impl WatcherInner +where + E: WatchEvents, +{ + /// Get a reference to the watcher inner's current value. + pub(crate) fn current_value(&self) -> &E::Value { + &self.current_value + } + + /// Creates a watcher subscription, which holds a snapshot of the current value, and a receiver which + /// will receive a stream of events as the watched key is updated. + pub(crate) fn create_subscription(&mut self) -> WatcherSubscription { + self.stats.subscriptions_created.incr(); + self.create_subscription_impl() + } + + fn create_subscription_impl(&mut self) -> WatcherSubscription { + let subscription_id = WatcherSubscriptionId(self.next_subscription_id); + self.next_subscription_id += 1; + let (sender, receiver) = mpsc::channel(E::WATCHER_MAX_BACKLOG_SIZE); + self.watchers.insert(subscription_id, sender); + WatcherSubscription { + subscription_id, + receiver, + full_sync_event: E::full_sync_event(self.current_value.clone()), + } + } + + fn update_subscribers(&mut self, events: Vec, current_value: &::Value) { + self.current_value = current_value.clone(); + + // No watchers or events, so no further dispatching to do. + if self.watchers.is_empty() || events.is_empty() { + return; + } + + // We have events to dispatch to their respective watchers. + for event in events { + let prev_watchers_len = self.watchers.len(); + + // Try to write to all senders, retaining only the ones that have not overflowed. The ones that have overflowed will need to be + // restarted, as they missed events. This will trigger a full-sync. This is to make sure if a receiver goes pathologically + // behind, we'll terminate it, and it must restart watching. + self.watchers + .retain(|_, sender| sender.try_send(event.clone()).is_ok()); + + let watchers_len = self.watchers.len(); + let watchers_removed = prev_watchers_len - watchers_len; + self.stats + .subscriptions_reset + .incr_by(watchers_removed as _); + self.stats.subscriptions_notified.incr_by(watchers_len as _); + } + } + + pub(crate) fn remove_subscription(&mut self, watcher_id: WatcherSubscriptionId) { + if self.watchers.remove(&watcher_id).is_some() { + self.stats.subscriptions_removed.incr(); + } + } + + pub(crate) fn recreate_subscription( + &mut self, + watcher_id: WatcherSubscriptionId, + ) -> WatcherSubscription { + self.stats.subscriptions_recreated.incr(); + self.watchers.remove(&watcher_id); + self.create_subscription_impl() + } + + pub(crate) fn num_subscribers(&self) -> usize { + self.watchers.len() + } +} + +pub(crate) struct WatcherLoop +where + I: WatcherImpl, +{ + inner: Arc>>, + watcher_impl: I, + backoff: AsyncBackoff, + stats: Arc, +} + +impl WatcherLoop +where + I: WatcherImpl, +{ + /// Starts the watcher loop, resolving once the loop has started, and an initial value has been read from etcd. + pub(crate) async fn start( + watcher_impl: I, + stats: Arc, + ) -> Result, crate::etcd::EtcdError> { + // Begin watching by synchronizing with etcd. + let initial_watch_state = match watcher_impl.begin_watching().await { + Ok(initial_watch_state) => initial_watch_state, + Err(err) => { + error!( + "watcher {:?} failed to start (error: {:?})", + watcher_impl, err + ); + stats.watcher_start_fails.incr(); + return Err(err); + } + }; + + let key = watcher_impl.key().to_owned(); + let backoff = AsyncBackoff::new(AsyncBackoffConfig { + min_delay: Duration::from_secs(5), + max_delay: Duration::from_secs(60), + multiplier: 2.0, + }); + + let watcher = Self { + watcher_impl, + backoff, + stats: stats.clone(), + inner: Arc::new(RwLock::new(WatcherInner { + current_value: initial_watch_state.current_value.clone(), + next_subscription_id: 0, + watchers: HashMap::new(), + stats, + key, + })), + }; + + let inner = watcher.inner.clone(); + let join_handle = tokio::task::spawn(watcher.run_loop(initial_watch_state)); + let watch_handle = WatchHandle::new(inner, join_handle); + + Ok(watch_handle) + } + + /// The background task that runs the watcher loop fsm. + async fn run_loop( + mut self, + initial_watch_state: WatchState<::Value>, + ) { + let mut watch_loop_state = WatchLoopState::ContinueWatching(initial_watch_state); + self.stats.watcher_start_success.incr(); + info!("Watcher (impl={:?} has started)", &self.watcher_impl); + + loop { + let (next_watch_loop_state, events) = self.run_loop_once(watch_loop_state).await; + watch_loop_state = next_watch_loop_state; + self.inner + .write() + .update_subscribers(events, watch_loop_state.current_value()); + } + } + + /// Runs one iteration of the watcher loop, returning the events to be emitted, and the next watch loop state that this function will be called with. + async fn run_loop_once( + &mut self, + watch_loop_state: WatchLoopState<::Value>, + ) -> ( + WatchLoopState<::Value>, + Vec, + ) { + let (watch_result, prev_watch_state) = match watch_loop_state { + WatchLoopState::ContinueWatching(prev_watch_state) => { + let watch_result = self.watcher_impl.continue_watching(&prev_watch_state).await; + + match watch_result { + // The watch was a success! + Ok(result) => (result, prev_watch_state), + // The watch failed, we'll backoff and try again. + Err(err) => { + self.log_and_sleep_after_error(err).await; + return (WatchLoopState::ContinueWatching(prev_watch_state), vec![]); + } + } + } + WatchLoopState::ResetWatcher(prev_watch_state) => { + match self + .watcher_impl + .reset_watcher(&prev_watch_state.current_value) + .await + { + // The watch reset was successful, which means we can continue the watcher now. + Ok(watch_result) => (watch_result, prev_watch_state), + // Watch reset was unsuccessful, we'll need to retry resetting. + Err(err) => { + self.log_and_sleep_after_error(err).await; + return (WatchLoopState::ResetWatcher(prev_watch_state), vec![]); + } + } + } + }; + + self.backoff.succeed(); + match watch_result { + // The watch was successful, and state will be updated to the next state. + WatchResult::Success { events, state } => { + self.stats.watch_success.incr(); + let next_state = WatchLoopState::ContinueWatching(state); + (next_state, events) + } + // The watch has timed out, so we'll retry the watch with the same state as before. + WatchResult::Timeout => { + self.stats.watch_timeout.incr(); + (WatchLoopState::ContinueWatching(prev_watch_state), vec![]) + } + // The watch needs to be reset. Inner watcher state has not changed. + WatchResult::ResetWatcher => { + self.stats.watch_reset.incr(); + (WatchLoopState::ResetWatcher(prev_watch_state), vec![]) + } + } + } + + async fn log_and_sleep_after_error(&mut self, error: crate::etcd::EtcdError) { + let delay = self.backoff.fail(); + error!( + "Watcher for key {} encountered an etcd error ({:?}), will retry after {:?}", + self.watcher_impl.key(), + error, + delay + ); + self.stats.watch_etcd_errors.incr(); + sleep(delay).await; + } +} + +pub(crate) enum WatchLoopState { + ContinueWatching(WatchState), + ResetWatcher(WatchState), +} + +impl WatchLoopState { + fn current_value(&self) -> &V { + match self { + WatchLoopState::ResetWatcher(watch_state) => &watch_state.current_value, + WatchLoopState::ContinueWatching(watch_state) => &watch_state.current_value, + } + } +} diff --git a/osprey_coordinator/src/etcd_watcherd/watcher_impl/key_watcher.rs b/osprey_coordinator/src/etcd_watcherd/watcher_impl/key_watcher.rs new file mode 100644 index 0000000..b2671c8 --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/watcher_impl/key_watcher.rs @@ -0,0 +1,150 @@ +use std::{fmt::Debug, sync::Arc}; + +use super::{ + util::{err_is_watch_timeout, err_should_reset_watcher, jittered_watch_timeout}, + WatchEvents, WatchResult, WatchState, WatcherImpl, +}; + +/// These events are emitted by [`crate::WatcherEvents`] when using a single key watcher. +#[derive(Clone, Debug)] +pub enum KeyWatchEvents { + /// The key has been updated. + /// - Some(value): Means that the key has been set to to the given value. + /// - None: The key has been deleted. + FullSync { value: Option> }, +} + +impl super::sealed::Sealed for KeyWatchEvents {} + +impl WatchEvents for KeyWatchEvents { + type Value = Option>; + const WATCHER_MAX_BACKLOG_SIZE: usize = 16; + + fn full_sync_event(value: Self::Value) -> Self { + Self::FullSync { value } + } +} + +pub(crate) struct KeyWatcher { + client: crate::etcd::Client, + key: String, +} + +impl Debug for KeyWatcher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("KeyWatcher") + .field("key", &self.key) + .finish() + } +} + +impl KeyWatcher { + pub(crate) fn new(client: crate::etcd::Client, key: String) -> Self { + Self { client, key } + } + + fn compute_delta_event( + current_value: &Option>, + new_value: Option>, + ) -> (Vec, Option>) { + // Only emit an event if the value actually changed. Otherwise, re-use the old value's arc. + if new_value != *current_value { + ( + vec![KeyWatchEvents::full_sync_event(new_value.clone())], + new_value, + ) + } else { + (vec![], current_value.clone()) + } + } +} + +#[async_trait::async_trait] +impl WatcherImpl for KeyWatcher { + type Events = KeyWatchEvents; + + fn key(&self) -> &str { + &self.key + } + + async fn begin_watching( + &self, + ) -> Result::Value>, crate::etcd::EtcdError> { + let result = self.client.get(self.key()).await; + let response = match result { + // Handle key-not-found by setting the initial value to None. + Err(crate::etcd::EtcdError::KeyNotFound { index, .. }) => { + return Ok(WatchState { + current_value: None, + wait_index: index, + }) + } + Err(etcd_error) => return Err(etcd_error), + Ok(ok) => ok, + }; + + Ok(WatchState { + current_value: response.data.node.value.map(|s| s.into()), + wait_index: response.cluster_info.etcd_index.unwrap_or_default(), + }) + } + + async fn continue_watching( + &self, + watch_state: &WatchState<::Value>, + ) -> Result, crate::etcd::EtcdError> { + let result = self + .client + .watch_opt( + self.key(), + watch_state.wait_index, + false, + jittered_watch_timeout(), + ) + .await; + + let (new_value, wait_index) = match result { + // If the error is a event index cleared code, we need to reset the watcher. + Err(ref err) if err_should_reset_watcher(err) => return Ok(WatchResult::ResetWatcher), + // The error is a timeout, so we should propegate that upstream. + Err(ref err) if err_is_watch_timeout(err) => return Ok(WatchResult::Timeout), + // A generic etcd error can be propagated to the caller. + Err(etcd_error) => return Err(etcd_error), + // Otherwise, we have got a value for the key. + Ok(ok) => ( + ok.data.node.value, + ok.data.node.modified_index.unwrap_or_default(), + ), + }; + + let (events, current_value) = + Self::compute_delta_event(&watch_state.current_value, new_value.map(|s| s.into())); + + Ok(WatchResult::Success { + events, + state: WatchState { + current_value, + wait_index, + }, + }) + } + + async fn reset_watcher( + &self, + current_value: &::Value, + ) -> Result, crate::etcd::EtcdError> { + let WatchState { + wait_index, + current_value: next_value, + } = self.begin_watching().await?; + let (events, current_value) = Self::compute_delta_event(current_value, next_value); + + Ok(WatchResult::Success { + events, + state: WatchState { + current_value, + wait_index, + }, + }) + } +} diff --git a/osprey_coordinator/src/etcd_watcherd/watcher_impl/mod.rs b/osprey_coordinator/src/etcd_watcherd/watcher_impl/mod.rs new file mode 100644 index 0000000..c105eba --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/watcher_impl/mod.rs @@ -0,0 +1,67 @@ +pub(crate) mod key_watcher; +pub(crate) mod recursive_key_watcher; +pub(self) mod util; + +use std::fmt::Debug; + +pub(crate) mod sealed { + // sealed trait pattern, no one can impl `WatchEvents` outside of this crate. + pub trait Sealed {} +} + +pub trait WatchEvents: sealed::Sealed + Clone + Send + Sync + 'static { + type Value: Sync + Send + Clone + 'static; + const WATCHER_MAX_BACKLOG_SIZE: usize; + + fn full_sync_event(value: Self::Value) -> Self; +} + +/// This trait basically powers the finite state machine of watching an etcd key. It is executed by [`crate::WatcherLoop`]. +/// +/// Implementators are the key watcher and recursive key watcher. +#[async_trait::async_trait] +pub(crate) trait WatcherImpl: Debug + Send + Sync + 'static { + type Events: WatchEvents; + + fn key(&self) -> &str; + + /// Starts the watch phase, gets the current watched value, and the etcd index, see [`WatchState`]. + async fn begin_watching( + &self, + ) -> Result::Value>, crate::etcd::EtcdError>; + + /// When etcd returns an error that the watcher has fallen too far behind, we need to reset the watcher. This method + /// is invoked when that is called. See [`WatchResult`] for more info. + async fn reset_watcher( + &self, + current_value: &::Value, + ) -> Result, crate::etcd::EtcdError>; + + /// This is invoked with the [`WatchState`] and is called when we should continue to long-poll etcd for changes to the key. + async fn continue_watching( + &self, + watch_state: &WatchState<::Value>, + ) -> Result, crate::etcd::EtcdError>; +} + +/// Holds the current value of the watcher, and the etcd index that we should continue to call the watch call on. +pub(crate) struct WatchState { + pub current_value: V, + pub wait_index: u64, +} + +/// Returned by the `continue_watching` and `reset_watcher` methods on [`WatcherImpl`]. +pub(crate) enum WatchResult +where + E: WatchEvents, +{ + /// The watch was successful, and we should call [`WatcherImpl::continue_watching`] next. + Success { + events: Vec, + state: WatchState, + }, + /// The watch had timed out, and we should call [`WatcherImpl::continue_watching`] next, not advancing the state. + Timeout, + /// The watch needs to be reset, and we should call [`WatcherImpl::reset_watcher`] next. + ResetWatcher, +} diff --git a/osprey_coordinator/src/etcd_watcherd/watcher_impl/recursive_key_watcher.rs b/osprey_coordinator/src/etcd_watcherd/watcher_impl/recursive_key_watcher.rs new file mode 100644 index 0000000..e7df183 --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/watcher_impl/recursive_key_watcher.rs @@ -0,0 +1,402 @@ +use std::{ + collections::{HashMap, HashSet}, + fmt::Debug, + sync::Arc, +}; + +use crate::etcd::Action; +use tracing::warn; + +use super::{ + util::{err_is_watch_timeout, err_should_reset_watcher, jittered_watch_timeout}, + WatchEvents, WatchResult, WatchState, WatcherImpl, +}; + +/// These events are emitted by [`crate::WatcherEvents`] when using a recursive watcher. +#[derive(Clone, Debug)] +pub enum RecursiveKeyWatchEvents { + /// A FullSync event is emitted as the initial watch event, but also, can be emitted after the initial watch event. + /// + /// This event represents a "snapshot" of the data within the etcd key watcher. Sometimes, the key watcher + /// might reset internally and decide to send a full sync again, so you should be prepared to handle this event multiple times. + /// + /// Additionally, + /// - this event can be emitted with an empty `items` hashmap, if the directory that is being watched is deleted. + /// - this event can be emitted if your watcher falls too far behind in handling + /// [`RecursiveKeyWatchEvents::SyncOne`] or [`RecursiveKeyWatchEvents::DeleteOne`] events, + /// to catch up the watcher. + FullSync { + items: Arc, Arc>>, + }, + /// Emitted when a single key is either created or updated. + SyncOne { key: Arc, value: Arc }, + /// Emitted when a single key is deleted or expires. + DeleteOne { key: Arc, prev_value: Arc }, +} + +impl super::sealed::Sealed for RecursiveKeyWatchEvents {} + +impl WatchEvents for RecursiveKeyWatchEvents { + type Value = Arc, Arc>>; + const WATCHER_MAX_BACKLOG_SIZE: usize = 64; + + fn full_sync_event(value: Self::Value) -> Self { + Self::FullSync { items: value } + } +} + +pub(crate) struct RecursiveKeyWatcher { + client: crate::etcd::Client, + key: String, +} + +impl Debug for RecursiveKeyWatcher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RecursiveKeyWatcher") + .field("key", &self.key) + .finish() + } +} + +impl RecursiveKeyWatcher { + pub(crate) fn new(client: crate::etcd::Client, key: String) -> Self { + Self { client, key } + } +} + +type Value = ::Value; + +#[async_trait::async_trait] +impl WatcherImpl for RecursiveKeyWatcher { + type Events = RecursiveKeyWatchEvents; + + fn key(&self) -> &str { + &self.key + } + + async fn begin_watching(&self) -> Result, crate::etcd::EtcdError> { + let result = self.client.get_recursive(self.key()).await; + let response = match result { + // Key does not exist, so we'll start out with an empty directory. + Err(crate::etcd::EtcdError::KeyNotFound { index, .. }) => { + return Ok(WatchState { + current_value: Default::default(), + wait_index: index, + }) + } + Err(etcd_error) => return Err(etcd_error), + Ok(ok) => ok, + }; + + let wait_index = response.cluster_info.etcd_index.unwrap_or_default(); + let current_value = Self::collect_node_children(response.data.node); + + Ok(WatchState { + wait_index, + current_value, + }) + } + + async fn reset_watcher( + &self, + current_value: &Value, + ) -> Result, crate::etcd::EtcdError> { + let watch_state = self.begin_watching().await?; + let wait_index = watch_state.wait_index; + + // The directory is empty. Figure out whether or not we should dispatch events. + if watch_state.current_value.is_empty() { + // If the current value is also empty, we can emit no events, as nothing has changed (empty is empty), + // additionally, we can continue to use the old `current_value`. + let (current_value, events) = if current_value.is_empty() { + (current_value.clone(), vec![]) + } else { + // Otherwise, current value was not empty. We'll emit a `FullSync` to notify all consumers to clear their copy of the data. + ( + watch_state.current_value.clone(), + vec![RecursiveKeyWatchEvents::FullSync { + items: watch_state.current_value, + }], + ) + }; + + return Ok(WatchResult::Success { + events, + state: WatchState { + current_value, + wait_index, + }, + }); + } + + let initial_value = Arc::try_unwrap(watch_state.current_value) + .expect("invariant: arc has more than 1 reference"); + + // We have data, and will do a diff. + let (events, current_value) = Self::compute_reset_events(current_value, initial_value); + + Ok(WatchResult::Success { + events, + state: WatchState { + wait_index, + current_value, + }, + }) + } + + async fn continue_watching( + &self, + watch_state: &WatchState, + ) -> Result, crate::etcd::EtcdError> { + let result = self + .client + .watch_opt( + self.key(), + watch_state.wait_index, + true, + jittered_watch_timeout(), + ) + .await; + + let data = match result { + // If the error is a event index cleared code, we need to reset the watcher. + Err(ref err) if err_should_reset_watcher(err) => return Ok(WatchResult::ResetWatcher), + // The error is a timeout, so we should propegate that upstream. + Err(ref err) if err_is_watch_timeout(err) => return Ok(WatchResult::Timeout), + // A generic etcd error can be propagated to the caller. + Err(etcd_error) => return Err(etcd_error), + // Otherwise, we have got a value for the key. + Ok(ok) => ok.data, + }; + let wait_index = data.node.modified_index.unwrap_or_default(); + + match data.action { + // Key has been deleted. + Action::CompareAndDelete | Action::Delete | Action::Expire => { + let dir = data.node.dir.unwrap_or_default(); + let key = data.node.key.unwrap_or_default(); + let (events, current_value) = if dir { + self.compute_dir_delete_events(key, watch_state) + } else { + self.compute_single_key_delete_events(key, watch_state) + }; + + Ok(WatchResult::Success { + events, + state: WatchState { + current_value, + wait_index, + }, + }) + } + // Key has been set. + Action::Create | Action::Set | Action::Update | Action::CompareAndSwap => { + let key = data.node.key.unwrap_or_default(); + let value = data.node.value.unwrap_or_default(); + let (events, current_value) = + self.compute_key_update_events(watch_state, key, value); + + Ok(WatchResult::Success { + events, + state: WatchState { + current_value, + wait_index, + }, + }) + } + // We really shouldn't get GET actions. If we do, let's just reset the watcher, and start over. + Action::Get => { + warn!( + "Watch for key {} returned an unexpected action {:?}, will reset watcher.", + self.key(), + data.action, + ); + Ok(WatchResult::ResetWatcher) + } + } + } +} + +impl RecursiveKeyWatcher { + /// Compute the events that need to be emitted if a directory is deleted. + fn compute_dir_delete_events( + &self, + key: String, + watch_state: &WatchState, + ) -> (Vec, Value) { + // The root key we were watching has been deleted, so we'll emit a full sync. + if &key == self.key() { + // Current value was also empty, so we can just re-use it, and emit no events. + if watch_state.current_value.is_empty() { + (vec![], watch_state.current_value.clone()) + } else { + // Otherwise, we'll create an empty map to store as our current value, emit a full sync, + // to signify the directory is empty. + let empty = Arc::new(HashMap::new()); + ( + vec![RecursiveKeyWatchEvents::FullSync { + items: empty.clone(), + }], + empty, + ) + } + } else { + // Otherwise, we've got a sub key we're deleting. We'll go ahead and loop over all keys and remove the ones we care about. + // NOTE: This could probably be made much more efficient if we backed this with a BTreeMap internally, but for our use-cases, + // the watched directories are not too big, and sub-directory deletes are quite rare. + let mut cloned = HashMap::clone(&watch_state.current_value); + let prefix = format!("{}/", key); + let mut events = vec![]; + cloned.retain(|k, v| { + if k.starts_with(&prefix) { + events.push(RecursiveKeyWatchEvents::DeleteOne { + key: k.clone(), + prev_value: v.clone(), + }); + false + } else { + true + } + }); + + (events, cloned.into()) + } + } + + /// Compute the events that need to be emitted if a single key is deleted. + fn compute_single_key_delete_events( + &self, + key: String, + watch_state: &WatchState, + ) -> (Vec, Value) { + // We're removing an individual key. + let mut cloned = HashMap::clone(&watch_state.current_value); + if let Some((key, prev_value)) = cloned.remove_entry(key.as_str()) { + // Recycle the Arc for that individual key in the `DeleteOne` event. + ( + vec![RecursiveKeyWatchEvents::DeleteOne { key, prev_value }], + cloned.into(), + ) + } else { + // This is an edge-case, and shold really never happen, but if it does, we'll throw + // away the clone we just made, and re-use the current value's arc. + (vec![], watch_state.current_value.clone()) + } + } + + /// Given the current [`WatchState`], and a key/value pair, compute the events that need to be emitted, + /// and also, return a new [`Value`] with the mutation applied. + fn compute_key_update_events( + &self, + watch_state: &WatchState, + key: String, + value: String, + ) -> (Vec, Value) { + match watch_state.current_value.get_key_value(key.as_str()) { + // Value has not changed, even though it was "set" in etcd, we drop events that don't effectively change the value. + // Additionally, we'll re-use the underlying watch_state's current value, by cloning the Arc, instead of doing a shallow copy. + Some((_, current_value)) if **current_value == value => { + (vec![], watch_state.current_value.clone()) + } + // Value has changed, or doesn't exist yet. + other => { + let mut cloned = HashMap::clone(&watch_state.current_value); + // Try and recycle the current key's arc. + let key = match other { + Some((key, _)) => key.clone(), + None => key.into(), + }; + let value: Arc = value.into(); + cloned.insert(key.clone(), value.clone()); + let events = vec![RecursiveKeyWatchEvents::SyncOne { key, value }]; + + (events, cloned.into()) + } + } + } + + /// When a reset is received, we'll perform a simple diff of the current value, and the latest value as returned by etcd. + /// + /// This function computes the events that need to be emitted. + fn compute_reset_events( + current_value: &Value, + next_value: HashMap, Arc>, + ) -> (Vec, Value) { + let mut events = vec![]; + let mut maybe_next_value = None; + let mut unvisited_keys: HashSet<_> = current_value.keys().collect(); + + // Compute events for all new/updated keys. + for (key, value) in next_value.into_iter() { + unvisited_keys.remove(&key); + + match current_value.get(&key) { + // The value is the same. + Some(current_key_value) if current_key_value == &value => continue, + _ => { + // We're about to mutate, so clone the current map, so we can mutate it. + let cloned = + maybe_next_value.get_or_insert_with(|| HashMap::clone(current_value)); + + events.push(RecursiveKeyWatchEvents::SyncOne { + key: key.clone(), + value: value.clone(), + }); + cloned.insert(key, value); + } + } + } + + // Compute events for all deleted keys. + for key in unvisited_keys { + let cloned = maybe_next_value.get_or_insert_with(|| HashMap::clone(current_value)); + if let Some((key, prev_value)) = cloned.remove_entry(key) { + events.push(RecursiveKeyWatchEvents::DeleteOne { key, prev_value }); + } + } + + let next_value = match maybe_next_value { + Some(cloned) => cloned.into(), + None => current_value.clone(), + }; + (events, next_value) + } + + /// Given an etcd node that was returned by a recursive get operation, descend down the node and its children, and collect + /// all values into a hash map ([`Value`]). + fn collect_node_children(node: crate::etcd::Node) -> Value { + /// Counts how values exist in a given node, recursively traversing down its children and summing those up too. + fn count_children(num_total_children: usize, node: &crate::etcd::Node) -> usize { + // If we are a directory, we need to count our children. + let num_node_children = if node.dir.unwrap_or_default() { + // Children can be `None` if there are no children, so we handle that case with the `unwrap_or_default()`. + node.nodes + .as_ref() + .map(|children| children.iter().fold(0, count_children)) + .unwrap_or_default() + } else { + 1 + }; + + num_node_children + num_total_children + } + + /// Recursively traverses down a `node`, populating `target` with the keys & values found. + fn collect_children(target: &mut HashMap, Arc>, node: crate::etcd::Node) { + if let Some(children) = node.nodes { + for child in children { + collect_children(target, child); + } + } else if let (Some(key), Some(value)) = (node.key, node.value) { + target.insert(key.into(), value.into()); + } + } + + let capacity = count_children(0, &node); + let mut value = HashMap::with_capacity(capacity); + collect_children(&mut value, node); + debug_assert_eq!(value.len(), capacity); + + value.into() + } +} diff --git a/osprey_coordinator/src/etcd_watcherd/watcher_impl/util.rs b/osprey_coordinator/src/etcd_watcherd/watcher_impl/util.rs new file mode 100644 index 0000000..70ac1f9 --- /dev/null +++ b/osprey_coordinator/src/etcd_watcherd/watcher_impl/util.rs @@ -0,0 +1,37 @@ +use std::time::Duration; + +use rand::{thread_rng, Rng}; + +/// Checks a given etcd error to see if it contains the special error code "event index cleared" +/// which signifies that the watcher should be reset. +pub(super) fn err_should_reset_watcher(err: &crate::etcd::EtcdError) -> bool { + const ETCD_ERROR_CODE_EVENT_INDEX_CLEARED: u64 = 401; + + match err { + crate::etcd::EtcdError::API { + error: crate::etcd::EtcdApiError::Api(api_error), + .. + } if api_error.error_code == ETCD_ERROR_CODE_EVENT_INDEX_CLEARED => true, + _ => false, + } +} + +/// Checks to see if the error is a watch timeout. +pub(super) fn err_is_watch_timeout(err: &crate::etcd::EtcdError) -> bool { + match err { + crate::etcd::EtcdError::Watch { + error: crate::etcd::WatchError::Timeout, + } => true, + _ => false, + } +} + +/// Returns a timeout, with jitter for how long we should watch the key, before timing out and trying again. +pub(super) fn jittered_watch_timeout() -> Duration { + // These timeouts are specifically chosen to be high, because the etcd client uses tcp_keepalive, and we can detect + // zombied connections using that. So we'll use a higher timeout here so we can hopefully reduce watcher churn. + const WATCH_TIMEOUT_MIN_MS: u64 = 600_000; + const WATCH_TIMEOUT_MAX_MS: u64 = 1_200_000; + + Duration::from_millis(thread_rng().gen_range(WATCH_TIMEOUT_MIN_MS..=WATCH_TIMEOUT_MAX_MS)) +} diff --git a/osprey_coordinator/src/future_utils/mod.rs b/osprey_coordinator/src/future_utils/mod.rs new file mode 100644 index 0000000..1ffb867 --- /dev/null +++ b/osprey_coordinator/src/future_utils/mod.rs @@ -0,0 +1,2 @@ +pub mod oneshot; +pub mod time; diff --git a/osprey_coordinator/src/future_utils/oneshot.rs b/osprey_coordinator/src/future_utils/oneshot.rs new file mode 100644 index 0000000..9bed5a6 --- /dev/null +++ b/osprey_coordinator/src/future_utils/oneshot.rs @@ -0,0 +1,23 @@ +use crate::future_utils::time::timeout_at; +use anyhow::{Error, Result}; +use futures::TryFutureExt; +use std::future::Future; +use std::marker::Unpin; +use std::time::Instant; +use tokio::sync::oneshot; + +/// Creates a oneshot that will time out at a deadline. +/// +/// # Arguments +/// +/// * deadline - the time to timeout at +pub fn channel_with_deadline( + deadline: Instant, +) -> (oneshot::Sender>, impl Future>) { + let (tx, rx) = oneshot::channel(); + let awaitable = async move { + let rx = rx.map_err(Error::from); + timeout_at(deadline, rx).await.unwrap_or_else(Err) + }; + (tx, awaitable) +} diff --git a/osprey_coordinator/src/future_utils/time.rs b/osprey_coordinator/src/future_utils/time.rs new file mode 100644 index 0000000..b1fa905 --- /dev/null +++ b/osprey_coordinator/src/future_utils/time.rs @@ -0,0 +1,15 @@ +use std::future::Future; +use std::io::{Error as IoError, ErrorKind}; +use std::time::Instant; + +/// Utility function for timing out a future without having to double unwrap the result. +/// It collapses the two results into a single result. +pub async fn timeout_at(deadline: Instant, future: F) -> F::Output +where + F: Future>, + E: From, +{ + tokio::time::timeout_at(deadline.into(), future) + .await + .unwrap_or_else(|_e| Err(IoError::new(ErrorKind::TimedOut, "timeout").into())) +} diff --git a/osprey_coordinator/src/gcloud/auth/auth_interceptor.rs b/osprey_coordinator/src/gcloud/auth/auth_interceptor.rs new file mode 100644 index 0000000..3a5d98f --- /dev/null +++ b/osprey_coordinator/src/gcloud/auth/auth_interceptor.rs @@ -0,0 +1,56 @@ +use std::sync::Arc; + +use http::header; +use tonic::{ + metadata::{errors::InvalidMetadataValue, Ascii, MetadataValue}, + service::Interceptor, +}; + +/// Indicates that the implementer can provide auth tokens for the auth headers +/// e.g. `Bearer abc.1231324dfasd` +pub trait AuthorizationHeaderProvider: Send + Sync + 'static { + fn get_auth_metadata(&self) -> Result, InvalidMetadataValue>; +} + +#[derive(Clone)] +pub struct AuthorizationHeaderInterceptor { + access_token_provider: Option>, +} + +impl AuthorizationHeaderInterceptor { + pub fn with_access_token_provider(access_token_provider: T) -> Self + where + T: AuthorizationHeaderProvider, + { + Self { + access_token_provider: Some(Arc::new(access_token_provider)), + } + } + + pub fn without_access_token_provider() -> Self { + Self { + access_token_provider: None, + } + } +} + +impl Interceptor for AuthorizationHeaderInterceptor { + fn call( + &mut self, + mut request: tonic::Request<()>, + ) -> Result, tonic::Status> { + if let Some(access_token_provider) = &self.access_token_provider { + match access_token_provider.get_auth_metadata() { + Ok(authorization_header) => { + request + .metadata_mut() + .insert(header::AUTHORIZATION.as_str(), authorization_header); + } + Err(err) => { + tracing::warn!("Failed to set authorization header: {}", err); + } + } + }; + Ok(request) + } +} diff --git a/osprey_coordinator/src/gcloud/auth/mod.rs b/osprey_coordinator/src/gcloud/auth/mod.rs new file mode 100644 index 0000000..be8a969 --- /dev/null +++ b/osprey_coordinator/src/gcloud/auth/mod.rs @@ -0,0 +1,9 @@ +mod auth_interceptor; +mod token; +mod token_refresher; + +pub use auth_interceptor::*; +pub use token::*; +pub use token_refresher::*; + +pub use goauth::scopes::Scope; diff --git a/osprey_coordinator/src/gcloud/auth/token.rs b/osprey_coordinator/src/gcloud/auth/token.rs new file mode 100644 index 0000000..2453ace --- /dev/null +++ b/osprey_coordinator/src/gcloud/auth/token.rs @@ -0,0 +1,107 @@ +use std::time::Instant; + +use serde::{self, Deserialize}; + +/// Represents an auth token for oauth +#[derive(Eq, PartialEq, Deserialize, Debug, Clone)] +pub struct Token { + access_token: String, + /// How long (in seconds) this token is valid. + #[serde(rename = "expires_in")] + pub expires_in_secs: u64, + pub token_type: String, + + /// Timestamp of when we created this instance, and when `expires_in` takes effect. + #[serde(skip, default = "Instant::now")] + pub created_at: Instant, +} + +/// Manually implement this so we don't print the `access_token` +impl std::fmt::Display for Token { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "Token: (type: {}, expires_in: {}s, alive_for: {}s)", + self.token_type, + self.expires_in_secs, + self.created_at.elapsed().as_secs(), + ) + } +} + +impl Token { + pub fn new( + access_token: String, + expires_in_secs: u64, + token_type: String, + created_at: Instant, + ) -> Self { + Self { + access_token, + expires_in_secs, + token_type, + created_at, + } + } + + pub fn from_goauth(token: goauth::auth::Token, created_at: Instant) -> Self { + Self { + access_token: token.access_token().to_string(), + expires_in_secs: token.expires_in() as u64, + token_type: token.token_type().to_string(), + created_at, + } + } + + pub fn is_expired(&self) -> bool { + let secs_since_created = self.created_at.elapsed().as_secs(); + secs_since_created > self.expires_in_secs + } + + /// Returns an active duration of how much time is left before the token expires. + /// `0` is returned if expired. + pub fn ttl_secs(&self) -> u64 { + let secs_since_created = self.created_at.elapsed().as_secs(); + + self.expires_in_secs.saturating_sub(secs_since_created) + } + + /// Returns a header value representation of the token + pub fn to_header_rep(&self) -> String { + format!("{} {}", self.token_type, self.access_token) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn test_token_expired() { + let token = Token { + access_token: "test.Sample".to_string(), + expires_in_secs: 3, + token_type: "Bearer".to_string(), + created_at: Instant::now() - Duration::from_secs(4), + }; + + assert!(token.is_expired()); + assert_eq!(token.ttl_secs(), 0, "time remaining should be 0"); + } + + #[test] + fn test_token_not_expired() { + let token = Token { + access_token: "test.Sample".to_string(), + expires_in_secs: 30, + token_type: "Bearer".to_string(), + created_at: Instant::now() - Duration::from_secs(1), + }; + + assert!(!token.is_expired()); + let ttl = token.ttl_secs(); + assert!(ttl >= 20 && ttl <= 29, "time remaining should be < expires"); + } +} diff --git a/osprey_coordinator/src/gcloud/auth/token_refresher.rs b/osprey_coordinator/src/gcloud/auth/token_refresher.rs new file mode 100644 index 0000000..1f11954 --- /dev/null +++ b/osprey_coordinator/src/gcloud/auth/token_refresher.rs @@ -0,0 +1,205 @@ +use crate::tokio_utils::AbortOnDrop; +use anyhow::{Context, Result}; +use arc_swap::ArcSwap; +use async_trait::async_trait; +use goauth::{auth::JwtClaims, credentials::Credentials}; +use smpl_jwt::Jwt; +use std::{ + cmp::max, + str::FromStr, + sync::Arc, + time::{Duration, Instant}, +}; +use tonic::metadata::{errors::InvalidMetadataValue, Ascii, MetadataValue}; +use which::which; + +use crate::gcloud::auth::Token; +use crate::gcloud::auth::{AuthorizationHeaderProvider, Scope}; + +#[async_trait] +pub trait TokenRefreshFn: Sync { + async fn get_token(&self) -> Result; +} + +#[derive(Clone)] +pub struct TokenRefresher { + token: Token, + refresher: T, +} + +/// A token refresher for using the gcloud command line to retrieve authentication credentials. +/// This shouldn't be used in production, but is very useful for testing things in development +pub(crate) struct GCloudAuth; + +#[async_trait] +impl TokenRefreshFn for GCloudAuth { + async fn get_token(&self) -> Result { + let gcloud_path = which("gcloud")?; + let output = tokio::process::Command::new(gcloud_path) + .args(["auth", "print-access-token", "--quiet"]) + .output() + .await?; + let stdout = String::from_utf8_lossy(&output.stdout); + let str_token = stdout.trim(); + let token = Token::new( + str_token.to_owned(), + 12 * 60 * 60, // 12 hours + "Bearer".into(), + Instant::now(), + ); + + tracing::info!("Token expires in {} seconds", token.ttl_secs()); + Ok(token) + } +} + +impl TokenRefresher { + pub async fn with_gcloud_auth() -> Result { + let initial = GCloudAuth.get_token().await?; + Ok(TokenRefresher::new(initial, GCloudAuth)) + } +} + +pub(crate) struct ApplicationCredentials { + credentials: Credentials, + scope: Scope, +} + +#[async_trait] +impl TokenRefreshFn for ApplicationCredentials { + async fn get_token(&self) -> Result { + tracing::info!("Requesting token for {:?} scope", self.scope); + let credentials = &self.credentials; + + let claims = JwtClaims::new( + credentials.iss(), + &self.scope, + credentials.token_uri(), + None, + None, + ); + let jwt = Jwt::new(claims, credentials.rsa_key().unwrap(), None); + + let token = goauth::get_token(&jwt, credentials) + .await + .context("Failed to refresh access token")?; + let token = Token::from_goauth(token, Instant::now()); + + tracing::info!("Token expires in {} seconds", token.ttl_secs()); + Ok(token) + } +} + +impl TokenRefresher { + fn load_credentials() -> Result { + let credentials_file = std::env::var("GOOGLE_APPLICATION_CREDENTIALS") + .context("GOOGLE_APPLICATION_CREDENTIALS environment variable not found")?; + + Credentials::from_file(&credentials_file) + .with_context(|| format!("Failed to read GCP credentials from {credentials_file}")) + } + + /// Helper to create a client using application-credentials contained with a slice containing json + pub async fn with_json_credentials(scope: Scope, data: &[u8]) -> Result { + let credentials: Credentials = serde_json::from_slice(data)?; + Self::with_credentials(scope, credentials).await + } + + async fn with_credentials(scope: Scope, credentials: Credentials) -> Result { + credentials.rsa_key().context("Invalid RSA key")?; + let refresher = ApplicationCredentials { credentials, scope }; + + let access_token_refresher = Self { + token: refresher + .get_token() + .await + .context("Initial credential load")?, + refresher, + }; + Ok(access_token_refresher) + } + + pub async fn with_local_credentials(scope: Scope) -> Result { + let credentials = Self::load_credentials()?; + Self::with_credentials(scope, credentials).await + } +} + +impl TokenRefresher +where + T: TokenRefreshFn + Send + 'static, +{ + pub fn new(token: Token, refresher: T) -> Self { + Self { token, refresher } + } + + /// Starts a background task to refresh the token every `refresh_period`. + /// This returns a `TokenHandler` that can be cloned and sent to clients; once + /// all the references are dropped this task is stopped. + pub fn spawn_refresher(mut self, refresh_period: Duration) -> TokenHandle { + let auth_header_value = Arc::new(ArcSwap::from(Arc::new(self.token.to_header_rep()))); + let refresher_auth_header_value = auth_header_value.clone(); + + let join_handle = tokio::spawn(async move { + tracing::info!("AccessTokenRefresher refresher started"); + loop { + let expires_in = self.token.ttl_secs(); + // Give ourselves some time before it actually expires + let expires_in = expires_in.saturating_sub(60); + + let min_refresh_period = if refresh_period.as_secs() < expires_in { + refresh_period + } else { + tracing::warn!( + "Token refresh interval was shorter than token expiration time: {}s", + expires_in + ); + // Sometimes "refreshing" the token doesn't actually refresh anything (we just pull the token from + // a get request against GCP). + // Just in case GCP is refreshing automatically at the e.g. 30 seconds to expire mark, we should not + // spin here since expires_in will be 0 for 60 seconds before the token expires. + Duration::from_secs(max(expires_in, 1)) + }; + + self.refresh(&refresher_auth_header_value).await; + tokio::time::sleep(min_refresh_period).await; + } + }); + + TokenHandle { + auth_header_value, + _token_refresher_handle: AbortOnDrop::new(join_handle), + } + } + + /// Fetch a new instance of the [Token] and update the stored value + async fn refresh(&mut self, access_token_header_value: &Arc>) { + if self.token.created_at.elapsed().as_secs() < (self.token.expires_in_secs - 3) / 2 { + // No need to refresh token since we will wake again before token expires + return; + } + + tracing::debug!("Refreshing token"); + let new_token = self.refresher.get_token().await; + match new_token { + Ok(new_token) => { + access_token_header_value.store(Arc::new(new_token.to_header_rep())); + self.token = new_token; + } + Err(err) => tracing::warn!("{}", err), + } + } +} + +pub struct TokenHandle { + pub(crate) auth_header_value: Arc>, + /// Held to ensure that the refresh task lives only as long as this client exists + pub(crate) _token_refresher_handle: AbortOnDrop<()>, +} + +impl AuthorizationHeaderProvider for TokenHandle { + fn get_auth_metadata(&self) -> Result, InvalidMetadataValue> { + let value = self.auth_header_value.load(); + FromStr::from_str(&value) + } +} diff --git a/osprey_coordinator/src/gcloud/gcp_metadata.rs b/osprey_coordinator/src/gcloud/gcp_metadata.rs new file mode 100644 index 0000000..a189104 --- /dev/null +++ b/osprey_coordinator/src/gcloud/gcp_metadata.rs @@ -0,0 +1,73 @@ +use anyhow::{Context, Result}; +use async_trait::async_trait; +use reqwest::Client; + +use crate::gcloud::auth::{Token, TokenRefreshFn, TokenRefresher}; + +/// Taken from https://cloud.google.com/appengine/docs/standard/java/accessing-instance-metadata +const METADATA_URL: &str = "http://metadata.google.internal/computeMetadata/v1"; +static APP_USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),); + +/// Provides helpers to get data about a GCP instance. This is done through accessing a http endpoint +/// that contains metadata about the current service. +/// +/// See https://cloud.google.com/appengine/docs/standard/java/accessing-instance-metadata +#[derive(Debug, Clone)] +pub struct GCPMetadataClient { + client: Client, + service_account: String, +} + +impl GCPMetadataClient { + pub fn new(service_account: String) -> Result { + let client = Client::builder().user_agent(APP_USER_AGENT).build()?; + Ok(Self { + client, + service_account, + }) + } + + pub async fn get_access_token(&self) -> Result { + let url = format!( + "{}/instance/service-accounts/{}/token", + METADATA_URL, self.service_account + ); + + /* + We should see something like this: + { + "access_token":"ya29.AHES6ZRN3-HlhAPya30GnW_bHSb_QtAS08i85nHq39HE3C2LTrCARA", + "expires_in":3599, + "token_type":"Bearer" + } + */ + let access_token = self + .client + .get(url) + .header("Metadata-Flavor", "Google") + .send() + .await? + .json::() + .await?; + Ok(access_token) + } +} + +#[async_trait] +impl TokenRefreshFn for GCPMetadataClient { + async fn get_token(&self) -> Result { + self.get_access_token().await + } +} + +impl TokenRefresher { + pub async fn with_metadata_client(client: GCPMetadataClient) -> Result { + Ok(Self::new( + client + .get_access_token() + .await + .context("Initial credential load")?, + client, + )) + } +} diff --git a/osprey_coordinator/src/gcloud/grpc/connection.rs b/osprey_coordinator/src/gcloud/grpc/connection.rs new file mode 100644 index 0000000..587995e --- /dev/null +++ b/osprey_coordinator/src/gcloud/grpc/connection.rs @@ -0,0 +1,147 @@ +//! Wrappers around making an authenticated GRPC connection to a Google Cloud service. + +use std::time::Duration; + +use thiserror::Error; +use tonic::transport::ClientTlsConfig; + +use crate::gcloud::{ + auth::{AuthorizationHeaderInterceptor, Scope, TokenRefresher}, + gcp_metadata::GCPMetadataClient, + root_ca_certificate, +}; + +#[derive(Debug, Error)] +pub enum ConnectionError { + #[error("AccessToken error: {0}")] + AccessTokenError(anyhow::Error), + + #[error("Certificate error: {0}")] + CertificateError(String), + + #[error("I/O Error: {0}")] + IoError(#[from] std::io::Error), + + #[error("Invalid URI {0}")] + InvalidUri(#[from] http::uri::InvalidUri), + + #[error("Transport error: {0}")] + TransportError(#[from] tonic::transport::Error), +} + +pub type Result = std::result::Result; + +#[derive(Clone)] +pub struct Connection { + pub(crate) authorization_header_interceptor: AuthorizationHeaderInterceptor, + pub(crate) channel: tonic::transport::Channel, +} + +impl Connection { + fn create_google_endpoint(domain: &str) -> Result { + let endpoint = tonic::transport::Channel::from_shared(format!("https://{domain}"))?; + let tls_config = ClientTlsConfig::new() + .ca_certificate(root_ca_certificate::load().map_err(ConnectionError::CertificateError)?) + .domain_name(domain); + endpoint.tls_config(tls_config).map_err(|e| e.into()) + } + + pub async fn from_metadata_client( + client: GCPMetadataClient, + timeout: impl Into>, + token_refresh_period: Duration, + domain: &str, + ) -> Result { + let access_token_handle = TokenRefresher::with_metadata_client(client) + .await + .map_err(ConnectionError::AccessTokenError)? + .spawn_refresher(token_refresh_period); + + let connection = Self { + authorization_header_interceptor: + AuthorizationHeaderInterceptor::with_access_token_provider(access_token_handle), + channel: apply_timeout(Self::create_google_endpoint(domain)?, timeout.into()), + }; + Ok(connection) + } + + pub async fn from_json( + timeout: impl Into>, + token_refresh_period: Duration, + domain: &str, + ac_json: &[u8], + scope: Scope, + ) -> Result { + let access_token_handle = TokenRefresher::with_json_credentials(scope, ac_json) + .await + .map_err(ConnectionError::AccessTokenError)? + .spawn_refresher(token_refresh_period); + + let connection = Self { + authorization_header_interceptor: + AuthorizationHeaderInterceptor::with_access_token_provider(access_token_handle), + channel: apply_timeout(Self::create_google_endpoint(domain)?, timeout.into()), + }; + Ok(connection) + } + + /// Create a new connection using gcloud cli authentication + pub async fn new_with_gcloud( + timeout: impl Into>, + token_refresh_period: Duration, + domain: &str, + ) -> Result { + let access_token_handle = TokenRefresher::with_gcloud_auth() + .await + .map_err(ConnectionError::AccessTokenError)? + .spawn_refresher(token_refresh_period); + + let connection = Self { + authorization_header_interceptor: + AuthorizationHeaderInterceptor::with_access_token_provider(access_token_handle), + channel: apply_timeout(Self::create_google_endpoint(domain)?, timeout.into()), + }; + Ok(connection) + } + + pub async fn new( + timeout: impl Into>, + token_refresh_period: Duration, + domain: &str, + scope: Scope, + ) -> Result { + let access_token_handle = TokenRefresher::with_local_credentials(scope) + .await + .map_err(ConnectionError::AccessTokenError)? + .spawn_refresher(token_refresh_period); + + let connection = Self { + authorization_header_interceptor: + AuthorizationHeaderInterceptor::with_access_token_provider(access_token_handle), + channel: apply_timeout(Self::create_google_endpoint(domain)?, timeout.into()), + }; + Ok(connection) + } + + pub fn new_no_auth( + endpoint: tonic::transport::Endpoint, + timeout: impl Into>, + ) -> Self { + Self { + authorization_header_interceptor: + AuthorizationHeaderInterceptor::without_access_token_provider(), + channel: apply_timeout(endpoint, timeout.into()), + } + } +} + +fn apply_timeout( + endpoint: tonic::transport::Endpoint, + timeout: Option, +) -> tonic::transport::Channel { + match timeout { + Some(timeout) => endpoint.timeout(timeout), + None => endpoint, + } + .connect_lazy() +} diff --git a/osprey_coordinator/src/gcloud/grpc/mod.rs b/osprey_coordinator/src/gcloud/grpc/mod.rs new file mode 100644 index 0000000..b3b606b --- /dev/null +++ b/osprey_coordinator/src/gcloud/grpc/mod.rs @@ -0,0 +1 @@ +pub mod connection; diff --git a/osprey_coordinator/src/gcloud/kms.rs b/osprey_coordinator/src/gcloud/kms.rs new file mode 100644 index 0000000..98dbb5f --- /dev/null +++ b/osprey_coordinator/src/gcloud/kms.rs @@ -0,0 +1,98 @@ +use anyhow::{anyhow, Result}; +use base64::engine::{general_purpose, Engine as _}; +use prost::Message; +use tink_aead::subtle::AesGcm; +use tink_core::Aead; +use tonic::codegen::InterceptedService; + +use crate::gcloud::auth::AuthorizationHeaderInterceptor; +use crate::gcloud::google::crypto::tink::AesGcmKey; +use crate::gcloud::google::kms::v1 as proto; +use crate::gcloud::google::kms::v1::key_management_service_client::KeyManagementServiceClient; +use crate::gcloud::grpc::connection::Connection; + +pub const GOOGLE_KMS_DOMAIN: &str = "cloudkms.googleapis.com"; + +const GCP_PREFIX: &str = "gcp-kms://"; + +impl Connection { + pub fn create_kms_aes_gcm_envelope( + &self, + key_uri: String, + associated_data: Vec, + base64_encoded: bool, + ) -> Result { + let key_uri = key_uri + .strip_prefix(GCP_PREFIX) + .ok_or_else(|| anyhow!("key uri must start with {}", GCP_PREFIX))?; + Ok(AesGcmEnvelope { + kms_client: self.create_kms_client(), + key_uri: key_uri.into(), + associated_data, + base64_encoded, + }) + } + + fn create_kms_client( + &self, + ) -> KeyManagementServiceClient< + InterceptedService, + > { + KeyManagementServiceClient::with_interceptor( + self.channel.clone(), + self.authorization_header_interceptor.clone(), + ) + } +} + +/// Reimplements some logic from tink-rust (https://github.com/project-oak/tink-rust) because the library doesn't +/// work with async code. +pub struct AesGcmEnvelope { + kms_client: KeyManagementServiceClient< + InterceptedService, + >, + key_uri: String, + associated_data: Vec, + base64_encoded: bool, +} + +impl AesGcmEnvelope { + // TODO: implement encrypt + + pub async fn decrypt(&self, ciphertext: &[u8]) -> Result> { + let ciphertext_decoded = if self.base64_encoded { + Some(base64::engine::general_purpose::STANDARD.decode(ciphertext)?) + } else { + None + }; + + let ciphertext = ciphertext_decoded + .as_ref() + .map(Vec::as_slice) + .unwrap_or(ciphertext); + + let enc_dek_len = u32::from_be_bytes(ciphertext[..4].try_into()?) as usize; + let ciphertext = &ciphertext[4..]; + + let encrypted_dek = &ciphertext[..enc_dek_len]; + let payload = &ciphertext[enc_dek_len..]; + + let decrypt_req = proto::DecryptRequest { + name: self.key_uri.clone(), + ciphertext: encrypted_dek.to_vec(), + ..Default::default() + }; + + let mut client_clone = self.kms_client.clone(); + + let decrypt_res = client_clone.decrypt(decrypt_req).await?.into_inner(); + + let data_encryption_key = AesGcmKey::decode(decrypt_res.plaintext.as_slice())?; + let aes_gcm = AesGcm::new(&data_encryption_key.key_value) + .map_err(|e| anyhow!("failed initializing AesGcm key object: {:#?}", e))?; + + Ok(aes_gcm + .decrypt(payload, &self.associated_data) + .map_err(|e| anyhow!("failed decrypting payload: {:#?}", e))?) + } +} diff --git a/osprey_coordinator/src/gcloud/mod.rs b/osprey_coordinator/src/gcloud/mod.rs new file mode 100644 index 0000000..980ab79 --- /dev/null +++ b/osprey_coordinator/src/gcloud/mod.rs @@ -0,0 +1,68 @@ +use std::time::Duration; + +pub mod auth; +pub mod gcp_metadata; +pub mod grpc; +mod root_ca_certificate; + +pub mod kms; +pub mod pubsub; + +/// Common options for creating authenticated connections to Google Cloud APIs +pub struct ConnectionOptions { + /// Request timeout + pub timeout: Duration, + + /// Duration to wait before refreshing the access token used for talking to cloud APIs + pub token_refresh_period: Duration, +} + +impl Default for ConnectionOptions { + fn default() -> ConnectionOptions { + Self { + timeout: Duration::from_secs(5), + token_refresh_period: Duration::from_secs(240000), // 4 Hours + } + } +} + +pub mod google { + pub mod api { + tonic::include_proto!("google.api"); + } + + pub mod iam { + pub mod v1 { + tonic::include_proto!("google.iam.v1"); + } + } + + pub mod longrunning { + tonic::include_proto!("google.longrunning"); + } + + pub mod rpc { + tonic::include_proto!("google.rpc"); + } + pub mod r#type { + tonic::include_proto!("google.r#type"); + } + + pub mod crypto { + pub mod tink { + tonic::include_proto!("google.crypto.tink"); + } + } + + pub mod pubsub { + pub mod v1 { + tonic::include_proto!("google.pubsub.v1"); + } + } + + pub mod kms { + pub mod v1 { + tonic::include_proto!("google.cloud.kms.v1"); + } + } +} diff --git a/osprey_coordinator/src/gcloud/pki-goog-roots.pem b/osprey_coordinator/src/gcloud/pki-goog-roots.pem new file mode 100644 index 0000000..0a6808d --- /dev/null +++ b/osprey_coordinator/src/gcloud/pki-goog-roots.pem @@ -0,0 +1,1222 @@ +# Operating CA: DigiCert +# Issuer: CN=Baltimore CyberTrust Root O=Baltimore OU=CyberTrust +# Subject: CN=Baltimore CyberTrust Root O=Baltimore OU=CyberTrust +# Label: "Baltimore CyberTrust Root" +# Serial: 33554617 +# MD5 Fingerprint: ac:b6:94:a5:9c:17:e0:d7:91:52:9b:b1:97:06:a6:e4 +# SHA1 Fingerprint: d4:de:20:d0:5e:66:fc:53:fe:1a:50:88:2c:78:db:28:52:ca:e4:74 +# SHA256 Fingerprint: 16:af:57:a9:f6:76:b0:ab:12:60:95:aa:5e:ba:de:f2:2a:b3:11:19:d6:44:ac:95:cd:4b:93:db:f3:f2:6a:eb +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIEAgAAuTANBgkqhkiG9w0BAQUFADBaMQswCQYDVQQGEwJJ +RTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJlclRydXN0MSIwIAYD +VQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTAwMDUxMjE4NDYwMFoX +DTI1MDUxMjIzNTkwMFowWjELMAkGA1UEBhMCSUUxEjAQBgNVBAoTCUJhbHRpbW9y +ZTETMBEGA1UECxMKQ3liZXJUcnVzdDEiMCAGA1UEAxMZQmFsdGltb3JlIEN5YmVy +VHJ1c3QgUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKMEuyKr +mD1X6CZymrV51Cni4eiVgLGw41uOKymaZN+hXe2wCQVt2yguzmKiYv60iNoS6zjr +IZ3AQSsBUnuId9Mcj8e6uYi1agnnc+gRQKfRzMpijS3ljwumUNKoUMMo6vWrJYeK +mpYcqWe4PwzV9/lSEy/CG9VwcPCPwBLKBsua4dnKM3p31vjsufFoREJIE9LAwqSu +XmD+tqYF/LTdB1kC1FkYmGP1pWPgkAx9XbIGevOF6uvUA65ehD5f/xXtabz5OTZy +dc93Uk3zyZAsuT3lySNTPx8kmCFcB5kpvcY67Oduhjprl3RjM71oGDHweI12v/ye +jl0qhqdNkNwnGjkCAwEAAaNFMEMwHQYDVR0OBBYEFOWdWTCCR1jMrPoIVDaGezq1 +BE3wMBIGA1UdEwEB/wQIMAYBAf8CAQMwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3 +DQEBBQUAA4IBAQCFDF2O5G9RaEIFoN27TyclhAO992T9Ldcw46QQF+vaKSm2eT92 +9hkTI7gQCvlYpNRhcL0EYWoSihfVCr3FvDB81ukMJY2GQE/szKN+OMY3EU/t3Wgx +jkzSswF07r51XgdIGn9w/xZchMB5hbgF/X++ZRGjD8ACtPhSNzkE1akxehi/oCr0 +Epn3o0WC4zxe9Z2etciefC7IpJ5OCBRLbf1wbWsaY71k5h+3zvDyny67G7fyUIhz +ksLi4xaNmjICq44Y3ekQEe5+NauQrz4wlHrQMz2nZQ/1/I6eYs9HRCwBXbsdtTLS +R9I4LtD+gdwyah617jzV/OeBHRnDJELqYzmp +-----END CERTIFICATE----- + +# Operating CA: DigiCert +# Issuer: CN=Cybertrust Global Root O=Cybertrust, Inc +# Subject: CN=Cybertrust Global Root O=Cybertrust, Inc +# Label: "Cybertrust Global Root" +# Serial: 4835703278459682877484360 +# MD5 Fingerprint: 72:e4:4a:87:e3:69:40:80:77:ea:bc:e3:f4:ff:f0:e1 +# SHA1 Fingerprint: 5f:43:e5:b1:bf:f8:78:8c:ac:1c:c7:ca:4a:9a:c6:22:2b:cc:34:c6 +# SHA256 Fingerprint: 96:0a:df:00:63:e9:63:56:75:0c:29:65:dd:0a:08:67:da:0b:9c:bd:6e:77:71:4a:ea:fb:23:49:ab:39:3d:a3 +-----BEGIN CERTIFICATE----- +MIIDoTCCAomgAwIBAgILBAAAAAABD4WqLUgwDQYJKoZIhvcNAQEFBQAwOzEYMBYG +A1UEChMPQ3liZXJ0cnVzdCwgSW5jMR8wHQYDVQQDExZDeWJlcnRydXN0IEdsb2Jh +bCBSb290MB4XDTA2MTIxNTA4MDAwMFoXDTIxMTIxNTA4MDAwMFowOzEYMBYGA1UE +ChMPQ3liZXJ0cnVzdCwgSW5jMR8wHQYDVQQDExZDeWJlcnRydXN0IEdsb2JhbCBS +b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA+Mi8vRRQZhP/8NN5 +7CPytxrHjoXxEnOmGaoQ25yiZXRadz5RfVb23CO21O1fWLE3TdVJDm71aofW0ozS +J8bi/zafmGWgE07GKmSb1ZASzxQG9Dvj1Ci+6A74q05IlG2OlTEQXO2iLb3VOm2y +HLtgwEZLAfVJrn5GitB0jaEMAs7u/OePuGtm839EAL9mJRQr3RAwHQeWP032a7iP +t3sMpTjr3kfb1V05/Iin89cqdPHoWqI7n1C6poxFNcJQZZXcY4Lv3b93TZxiyWNz +FtApD0mpSPCzqrdsxacwOUBdrsTiXSZT8M4cIwhhqJQZugRiQOwfOHB3EgZxpzAY +XSUnpQIDAQABo4GlMIGiMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/ +MB0GA1UdDgQWBBS2CHsNesysIEyGVjJez6tuhS1wVzA/BgNVHR8EODA2MDSgMqAw +hi5odHRwOi8vd3d3Mi5wdWJsaWMtdHJ1c3QuY29tL2NybC9jdC9jdHJvb3QuY3Js +MB8GA1UdIwQYMBaAFLYIew16zKwgTIZWMl7Pq26FLXBXMA0GCSqGSIb3DQEBBQUA +A4IBAQBW7wojoFROlZfJ+InaRcHUowAl9B8Tq7ejhVhpwjCt2BWKLePJzYFa+HMj +Wqd8BfP9IjsO0QbE2zZMcwSO5bAi5MXzLqXZI+O4Tkogp24CJJ8iYGd7ix1yCcUx +XOl5n4BHPa2hCwcUPUf/A2kaDAtE52Mlp3+yybh2hO0j9n0Hq0V+09+zv+mKts2o +omcrUtW3ZfA5TGOgkXmTUg9U3YO7n9GPp1Nzw8v/MOx8BLjYRB+TX3EJIrduPuoc +A06dGiBh+4E37F78CkWr1+cXVdCg6mCbpvbjjFspwgZgFJ0tl0ypkxWdYcQBX0jW +WL1WMRJOEcgh4LMRkWXbtKaIOM5V +-----END CERTIFICATE----- + +# Operating CA: DigiCert +# Issuer: CN=DigiCert Assured ID Root CA O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Assured ID Root CA O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Assured ID Root CA" +# Serial: 17154717934120587862167794914071425081 +# MD5 Fingerprint: 87:ce:0b:7b:2a:0e:49:00:e1:58:71:9b:37:a8:93:72 +# SHA1 Fingerprint: 05:63:b8:63:0d:62:d7:5a:bb:c8:ab:1e:4b:df:b5:a8:99:b2:4d:43 +# SHA256 Fingerprint: 3e:90:99:b5:01:5e:8f:48:6c:00:bc:ea:9d:11:1e:e7:21:fa:ba:35:5a:89:bc:f1:df:69:56:1e:3d:c6:32:5c +-----BEGIN CERTIFICATE----- +MIIDtzCCAp+gAwIBAgIQDOfg5RfYRv6P5WD8G/AwOTANBgkqhkiG9w0BAQUFADBl +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv +b3QgQ0EwHhcNMDYxMTEwMDAwMDAwWhcNMzExMTEwMDAwMDAwWjBlMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl +cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgQ0EwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtDhXO5EOAXLGH87dg+XESpa7c +JpSIqvTO9SA5KFhgDPiA2qkVlTJhPLWxKISKityfCgyDF3qPkKyK53lTXDGEKvYP +mDI2dsze3Tyoou9q+yHyUmHfnyDXH+Kx2f4YZNISW1/5WBg1vEfNoTb5a3/UsDg+ +wRvDjDPZ2C8Y/igPs6eD1sNuRMBhNZYW/lmci3Zt1/GiSw0r/wty2p5g0I6QNcZ4 +VYcgoc/lbQrISXwxmDNsIumH0DJaoroTghHtORedmTpyoeb6pNnVFzF1roV9Iq4/ +AUaG9ih5yLHa5FcXxH4cDrC0kqZWs72yl+2qp/C3xag/lRbQ/6GW6whfGHdPAgMB +AAGjYzBhMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW +BBRF66Kv9JLLgjEtUYunpyGd823IDzAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYun +pyGd823IDzANBgkqhkiG9w0BAQUFAAOCAQEAog683+Lt8ONyc3pklL/3cmbYMuRC +dWKuh+vy1dneVrOfzM4UKLkNl2BcEkxY5NM9g0lFWJc1aRqoR+pWxnmrEthngYTf +fwk8lOa4JiwgvT2zKIn3X/8i4peEH+ll74fg38FnSbNd67IJKusm7Xi+fT8r87cm +NW1fiQG2SVufAQWbqz0lwcy2f8Lxb4bG+mRo64EtlOtCt/qMHt1i8b5QZ7dsvfPx +H2sMNgcWfzd8qVttevESRmCD1ycEvkvOl77DZypoEd+A5wwzZr8TDRRu838fYxAe ++o0bJW1sj6W3YQGx0qMmoRBxna3iw/nDmVG3KwcIzi7mULKn+gpFL6Lw8g== +-----END CERTIFICATE----- + +# Operating CA: DigiCert +# Issuer: CN=DigiCert Assured ID Root G2 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Assured ID Root G2 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Assured ID Root G2" +# Serial: 15385348160840213938643033620894905419 +# MD5 Fingerprint: 92:38:b9:f8:63:24:82:65:2c:57:33:e6:fe:81:8f:9d +# SHA1 Fingerprint: a1:4b:48:d9:43:ee:0a:0e:40:90:4f:3c:e0:a4:c0:91:93:51:5d:3f +# SHA256 Fingerprint: 7d:05:eb:b6:82:33:9f:8c:94:51:ee:09:4e:eb:fe:fa:79:53:a1:14:ed:b2:f4:49:49:45:2f:ab:7d:2f:c1:85 +-----BEGIN CERTIFICATE----- +MIIDljCCAn6gAwIBAgIQC5McOtY5Z+pnI7/Dr5r0SzANBgkqhkiG9w0BAQsFADBl +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv +b3QgRzIwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl +cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDZ5ygvUj82ckmIkzTz+GoeMVSA +n61UQbVH35ao1K+ALbkKz3X9iaV9JPrjIgwrvJUXCzO/GU1BBpAAvQxNEP4Htecc +biJVMWWXvdMX0h5i89vqbFCMP4QMls+3ywPgym2hFEwbid3tALBSfK+RbLE4E9Hp +EgjAALAcKxHad3A2m67OeYfcgnDmCXRwVWmvo2ifv922ebPynXApVfSr/5Vh88lA +bx3RvpO704gqu52/clpWcTs/1PPRCv4o76Pu2ZmvA9OPYLfykqGxvYmJHzDNw6Yu +YjOuFgJ3RFrngQo8p0Quebg/BLxcoIfhG69Rjs3sLPr4/m3wOnyqi+RnlTGNAgMB +AAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQW +BBTOw0q5mVXyuNtgv6l+vVa1lzan1jANBgkqhkiG9w0BAQsFAAOCAQEAyqVVjOPI +QW5pJ6d1Ee88hjZv0p3GeDgdaZaikmkuOGybfQTUiaWxMTeKySHMq2zNixya1r9I +0jJmwYrA8y8678Dj1JGG0VDjA9tzd29KOVPt3ibHtX2vK0LRdWLjSisCx1BL4Gni +lmwORGYQRI+tBev4eaymG+g3NJ1TyWGqolKvSnAWhsI6yLETcDbYz+70CjTVW0z9 +B5yiutkBclzzTcHdDrEcDcRjvq30FPuJ7KJBDkzMyFdA0G4Dqs0MjomZmWzwPDCv +ON9vvKO+KSAnq3T/EyJ43pdSVR6DtVQgA+6uwE9W3jfMw3+qBCe703e4YtsXfJwo +IhNzbM8m9Yop5w== +-----END CERTIFICATE----- + +# Operating CA: DigiCert +# Issuer: CN=DigiCert Assured ID Root G3 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Assured ID Root G3 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Assured ID Root G3" +# Serial: 15459312981008553731928384953135426796 +# MD5 Fingerprint: 7c:7f:65:31:0c:81:df:8d:ba:3e:99:e2:5c:ad:6e:fb +# SHA1 Fingerprint: f5:17:a2:4f:9a:48:c6:c9:f8:a2:00:26:9f:dc:0f:48:2c:ab:30:89 +# SHA256 Fingerprint: 7e:37:cb:8b:4c:47:09:0c:ab:36:55:1b:a6:f4:5d:b8:40:68:0f:ba:16:6a:95:2d:b1:00:71:7f:43:05:3f:c2 +-----BEGIN CERTIFICATE----- +MIICRjCCAc2gAwIBAgIQC6Fa+h3foLVJRK/NJKBs7DAKBggqhkjOPQQDAzBlMQsw +CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu +ZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3Qg +RzMwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQGEwJV +UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu +Y29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzMwdjAQBgcq +hkjOPQIBBgUrgQQAIgNiAAQZ57ysRGXtzbg/WPuNsVepRC0FFfLvC/8QdJ+1YlJf +Zn4f5dwbRXkLzMZTCp2NXQLZqVneAlr2lSoOjThKiknGvMYDOAdfVdp+CW7if17Q +RSAPWXYQ1qAk8C3eNvJsKTmjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ +BAQDAgGGMB0GA1UdDgQWBBTL0L2p4ZgFUaFNN6KDec6NHSrkhDAKBggqhkjOPQQD +AwNnADBkAjAlpIFFAmsSS3V0T8gj43DydXLefInwz5FyYZ5eEJJZVrmDxxDnOOlY +JjZ91eQ0hjkCMHw2U/Aw5WJjOpnitqM7mzT6HtoQknFekROn3aRukswy1vUhZscv +6pZjamVFkpUBtA== +-----END CERTIFICATE----- + +# Operating CA: DigiCert +# Issuer: CN=DigiCert Global Root CA O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Global Root CA O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Global Root CA" +# Serial: 10944719598952040374951832963794454346 +# MD5 Fingerprint: 79:e4:a9:84:0d:7d:3a:96:d7:c0:4f:e2:43:4c:89:2e +# SHA1 Fingerprint: a8:98:5d:3a:65:e5:e5:c4:b2:d7:d6:6d:40:c6:dd:2f:b1:9c:54:36 +# SHA256 Fingerprint: 43:48:a0:e9:44:4c:78:cb:26:5e:05:8d:5e:89:44:b4:d8:4f:96:62:bd:26:db:25:7f:89:34:a4:43:c7:01:61 +-----BEGIN CERTIFICATE----- +MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD +QTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAwMDAwMDBaMGExCzAJBgNVBAYTAlVT +MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j +b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvhEXLeqKTTo1eqUKKPC3eQyaKl7hLOllsB +CSDMAZOnTjC3U/dDxGkAV53ijSLdhwZAAIEJzs4bg7/fzTtxRuLWZscFs3YnFo97 +nh6Vfe63SKMI2tavegw5BmV/Sl0fvBf4q77uKNd0f3p4mVmFaG5cIzJLv07A6Fpt +43C/dxC//AH2hdmoRBBYMql1GNXRor5H4idq9Joz+EkIYIvUX7Q6hL+hqkpMfT7P +T19sdl6gSzeRntwi5m3OFBqOasv+zbMUZBfHWymeMr/y7vrTC0LUq7dBMtoM1O/4 +gdW7jVg/tRvoSSiicNoxBN33shbyTApOB6jtSj1etX+jkMOvJwIDAQABo2MwYTAO +BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA95QNVbR +TLtm8KPiGxvDl7I90VUwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUw +DQYJKoZIhvcNAQEFBQADggEBAMucN6pIExIK+t1EnE9SsPTfrgT1eXkIoyQY/Esr +hMAtudXH/vTBH1jLuG2cenTnmCmrEbXjcKChzUyImZOMkXDiqw8cvpOp/2PV5Adg +06O/nVsJ8dWO41P0jmP6P6fbtGbfYmbW0W5BjfIttep3Sp+dWOIrWcBAI+0tKIJF +PnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886UAb3LujEV0ls +YSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk +CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4= +-----END CERTIFICATE----- + +# Operating CA: DigiCert +# Issuer: CN=DigiCert Global Root G2 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Global Root G2 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Global Root G2" +# Serial: 4293743540046975378534879503202253541 +# MD5 Fingerprint: e4:a6:8a:c8:54:ac:52:42:46:0a:fd:72:48:1b:2a:44 +# SHA1 Fingerprint: df:3c:24:f9:bf:d6:66:76:1b:26:80:73:fe:06:d1:cc:8d:4f:82:a4 +# SHA256 Fingerprint: cb:3c:cb:b7:60:31:e5:e0:13:8f:8d:d3:9a:23:f9:de:47:ff:c3:5e:43:c1:14:4c:ea:27:d4:6a:5a:b1:cb:5f +-----BEGIN CERTIFICATE----- +MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH +MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT +MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j +b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI +2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx +1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ +q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz +tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ +vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP +BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV +5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY +1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4 +NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG +Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91 +8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe +pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl +MrY= +-----END CERTIFICATE----- + +# Operating CA: DigiCert +# Issuer: CN=DigiCert Global Root G3 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Global Root G3 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Global Root G3" +# Serial: 7089244469030293291760083333884364146 +# MD5 Fingerprint: f5:5d:a4:50:a5:fb:28:7e:1e:0f:0d:cc:96:57:56:ca +# SHA1 Fingerprint: 7e:04:de:89:6a:3e:66:6d:00:e6:87:d3:3f:fa:d9:3b:e8:3d:34:9e +# SHA256 Fingerprint: 31:ad:66:48:f8:10:41:38:c7:38:f3:9e:a4:32:01:33:39:3e:3a:18:cc:02:29:6e:f9:7c:2a:c9:ef:67:31:d0 +-----BEGIN CERTIFICATE----- +MIICPzCCAcWgAwIBAgIQBVVWvPJepDU1w6QP1atFcjAKBggqhkjOPQQDAzBhMQsw +CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu +ZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBHMzAe +Fw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVTMRUw +EwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20x +IDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEczMHYwEAYHKoZIzj0CAQYF +K4EEACIDYgAE3afZu4q4C/sLfyHS8L6+c/MzXRq8NOrexpu80JX28MzQC7phW1FG +fp4tn+6OYwwX7Adw9c+ELkCDnOg/QW07rdOkFFk2eJ0DQ+4QE2xy3q6Ip6FrtUPO +Z9wj/wMco+I+o0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAd +BgNVHQ4EFgQUs9tIpPmhxdiuNkHMEWNpYim8S8YwCgYIKoZIzj0EAwMDaAAwZQIx +AK288mw/EkrRLTnDCgmXc/SINoyIJ7vmiI1Qhadj+Z4y3maTD/HMsQmP3Wyr+mt/ +oAIwOWZbwmSNuJ5Q3KjVSaLtx9zRSX8XAbjIho9OjIgrqJqpisXRAL34VOKa5Vt8 +sycX +-----END CERTIFICATE----- + +# Operating CA: DigiCert +# Issuer: CN=DigiCert High Assurance EV Root CA O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert High Assurance EV Root CA O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert High Assurance EV Root CA" +# Serial: 3553400076410547919724730734378100087 +# MD5 Fingerprint: d4:74:de:57:5c:39:b2:d3:9c:85:83:c5:c0:65:49:8a +# SHA1 Fingerprint: 5f:b7:ee:06:33:e2:59:db:ad:0c:4c:9a:e6:d3:8f:1a:61:c7:dc:25 +# SHA256 Fingerprint: 74:31:e5:f4:c3:c1:ce:46:90:77:4f:0b:61:e0:54:40:88:3b:a9:a0:1e:d0:0b:a6:ab:d7:80:6e:d3:b1:18:cf +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j +ZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL +MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3 +LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug +RVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm ++9S75S0tMqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTW +PNt0OKRKzE0lgvdKpVMSOO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEM +xChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFB +Ik5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQNAQTXKFx01p8VdteZOE3 +hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUeh10aUAsg +EsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQF +MAMBAf8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaA +FLE+w2kD+L9HAdSYJhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3Nec +nzyIZgYIVyHbIUf4KmeqvxgydkAQV8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6z +eM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFpmyPInngiK3BD41VHMWEZ71jF +hS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkKmNEVX58Svnw2 +Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe +vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep ++OkuE6N36B9K +-----END CERTIFICATE----- + +# Operating CA: DigiCert +# Issuer: CN=DigiCert Trusted Root G4 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Trusted Root G4 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Trusted Root G4" +# Serial: 7451500558977370777930084869016614236 +# MD5 Fingerprint: 78:f2:fc:aa:60:1f:2f:b4:eb:c9:37:ba:53:2e:75:49 +# SHA1 Fingerprint: dd:fb:16:cd:49:31:c9:73:a2:03:7d:3f:c8:3a:4d:7d:77:5d:05:e4 +# SHA256 Fingerprint: 55:2f:7b:dc:f1:a7:af:9e:6c:e6:72:01:7f:4f:12:ab:f7:72:40:c7:8e:76:1a:c2:03:d1:d9:d2:0a:c8:99:88 +-----BEGIN CERTIFICATE----- +MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBi +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg +RzQwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBiMQswCQYDVQQGEwJV +UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu +Y29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQC/5pBzaN675F1KPDAiMGkz7MKnJS7JIT3y +ithZwuEppz1Yq3aaza57G4QNxDAf8xukOBbrVsaXbR2rsnnyyhHS5F/WBTxSD1If +xp4VpX6+n6lXFllVcq9ok3DCsrp1mWpzMpTREEQQLt+C8weE5nQ7bXHiLQwb7iDV +ySAdYyktzuxeTsiT+CFhmzTrBcZe7FsavOvJz82sNEBfsXpm7nfISKhmV1efVFiO +DCu3T6cw2Vbuyntd463JT17lNecxy9qTXtyOj4DatpGYQJB5w3jHtrHEtWoYOAMQ +jdjUN6QuBX2I9YI+EJFwq1WCQTLX2wRzKm6RAXwhTNS8rhsDdV14Ztk6MUSaM0C/ +CNdaSaTC5qmgZ92kJ7yhTzm1EVgX9yRcRo9k98FpiHaYdj1ZXUJ2h4mXaXpI8OCi +EhtmmnTK3kse5w5jrubU75KSOp493ADkRSWJtppEGSt+wJS00mFt6zPZxd9LBADM +fRyVw4/3IbKyEbe7f/LVjHAsQWCqsWMYRJUadmJ+9oCw++hkpjPRiQfhvbfmQ6QY +uKZ3AeEPlAwhHbJUKSWJbOUOUlFHdL4mrLZBdd56rF+NP8m800ERElvlEFDrMcXK +chYiCd98THU/Y+whX8QgUWtvsauGi0/C1kVfnSD8oR7FwI+isX4KJpn15GkvmB0t +9dmpsh3lGwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +hjAdBgNVHQ4EFgQU7NfjgtJxXWRM3y5nP+e6mK4cD08wDQYJKoZIhvcNAQEMBQAD +ggIBALth2X2pbL4XxJEbw6GiAI3jZGgPVs93rnD5/ZpKmbnJeFwMDF/k5hQpVgs2 +SV1EY+CtnJYYZhsjDT156W1r1lT40jzBQ0CuHVD1UvyQO7uYmWlrx8GnqGikJ9yd ++SeuMIW59mdNOj6PWTkiU0TryF0Dyu1Qen1iIQqAyHNm0aAFYF/opbSnr6j3bTWc +fFqK1qI4mfN4i/RN0iAL3gTujJtHgXINwBQy7zBZLq7gcfJW5GqXb5JQbZaNaHqa +sjYUegbyJLkJEVDXCLG4iXqEI2FCKeWjzaIgQdfRnGTZ6iahixTXTBmyUEFxPT9N +cCOGDErcgdLMMpSEDQgJlxxPwO5rIHQw0uA5NBCFIRUBCOhVMt5xSdkoF1BN5r5N +0XWs0Mr7QbhDparTwwVETyw2m+L64kW4I1NsBm9nVX9GtUw/bihaeSbSpKhil9Ie +4u1Ki7wb/UdKDd9nZn6yW0HQO+T0O/QEY+nvwlQAUaCKKsnOeMzV6ocEGLPOr0mI +r/OSmbaz5mEP0oUA51Aa5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1 +/YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tKG48BtieVU+i2iW1bvGjUI+iLUaJW+fCm +gKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP82Z+ +-----END CERTIFICATE----- + +# Operating CA: DigiCert +# Issuer: CN=GeoTrust Global CA O=GeoTrust Inc. +# Subject: CN=GeoTrust Global CA O=GeoTrust Inc. +# Label: "GeoTrust Global CA" +# Serial: 144470 +# MD5 Fingerprint: f7:75:ab:29:fb:51:4e:b7:77:5e:ff:05:3c:99:8e:f5 +# SHA1 Fingerprint: de:28:f4:a4:ff:e5:b9:2f:a3:c5:03:d1:a3:49:a7:f9:96:2a:82:12 +# SHA256 Fingerprint: ff:85:6a:2d:25:1d:cd:88:d3:66:56:f4:50:12:67:98:cf:ab:aa:de:40:79:9c:72:2d:e4:d2:b5:db:36:a7:3a +-----BEGIN CERTIFICATE----- +MIIDVDCCAjygAwIBAgIDAjRWMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT +MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i +YWwgQ0EwHhcNMDIwNTIxMDQwMDAwWhcNMjIwNTIxMDQwMDAwWjBCMQswCQYDVQQG +EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3Qg +R2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD9 +9BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdq +fnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDv +iS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU +1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+ +bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoW +MPRfwCvocWvk+QIDAQABo1MwUTAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTA +ephojYn7qwVkDBF9qn1luMrMTjAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1l +uMrMTjANBgkqhkiG9w0BAQUFAAOCAQEANeMpauUvXVSOKVCUn5kaFOSPeCpilKIn +Z57QzxpeR+nBsqTP3UEaBU6bS+5Kb1VSsyShNwrrZHYqLizz/Tt1kL/6cdjHPTfS +tQWVYrmm3ok9Nns4d0iXrKYgjy6myQzCsplFAMfOEVEiIuCl6rYVSAlk6l5PdPcF +PseKUgzbFbS9bZvlxrFUaKnjaZC2mqUPuLk/IH2uSrW4nOQdtqvmlKXBx4Ot2/Un +hw4EbNX/3aBd7YdStysVAq45pmp06drE57xNNB6pXE0zX5IJL4hmXXeXxx12E6nV +5fEWCRE11azbJHFwLJhWC9kXtNHjUStedejV0NxPNO3CBWaAocvmMw== +-----END CERTIFICATE----- + +# Operating CA: Entrust Datacard +# Issuer: CN=Entrust Root Certification Authority O=Entrust, Inc. OU=www.entrust.net/CPS is incorporated by reference/(c) 2006 Entrust, Inc. +# Subject: CN=Entrust Root Certification Authority O=Entrust, Inc. OU=www.entrust.net/CPS is incorporated by reference/(c) 2006 Entrust, Inc. +# Label: "Entrust Root Certification Authority" +# Serial: 1164660820 +# MD5 Fingerprint: d6:a5:c3:ed:5d:dd:3e:00:c1:3d:87:92:1f:1d:3f:e4 +# SHA1 Fingerprint: b3:1e:b1:b7:40:e3:6c:84:02:da:dc:37:d4:4d:f5:d4:67:49:52:f9 +# SHA256 Fingerprint: 73:c1:76:43:4f:1b:c6:d5:ad:f4:5b:0e:76:e7:27:28:7c:8d:e5:76:16:c1:e6:e6:14:1a:2b:2c:bc:7d:8e:4c +-----BEGIN CERTIFICATE----- +MIIEkTCCA3mgAwIBAgIERWtQVDANBgkqhkiG9w0BAQUFADCBsDELMAkGA1UEBhMC +VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xOTA3BgNVBAsTMHd3dy5lbnRydXN0 +Lm5ldC9DUFMgaXMgaW5jb3Jwb3JhdGVkIGJ5IHJlZmVyZW5jZTEfMB0GA1UECxMW +KGMpIDIwMDYgRW50cnVzdCwgSW5jLjEtMCsGA1UEAxMkRW50cnVzdCBSb290IENl +cnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA2MTEyNzIwMjM0MloXDTI2MTEyNzIw +NTM0MlowgbAxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMTkw +NwYDVQQLEzB3d3cuZW50cnVzdC5uZXQvQ1BTIGlzIGluY29ycG9yYXRlZCBieSBy +ZWZlcmVuY2UxHzAdBgNVBAsTFihjKSAyMDA2IEVudHJ1c3QsIEluYy4xLTArBgNV +BAMTJEVudHJ1c3QgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBALaVtkNC+sZtKm9I35RMOVcF7sN5EUFo +Nu3s/poBj6E4KPz3EEZmLk0eGrEaTsbRwJWIsMn/MYszA9u3g3s+IIRe7bJWKKf4 +4LlAcTfFy0cOlypowCKVYhXbR9n10Cv/gkvJrT7eTNuQgFA/CYqEAOwwCj0Yzfv9 +KlmaI5UXLEWeH25DeW0MXJj+SKfFI0dcXv1u5x609mhF0YaDW6KKjbHjKYD+JXGI +rb68j6xSlkuqUY3kEzEZ6E5Nn9uss2rVvDlUccp6en+Q3X0dgNmBu1kmwhH+5pPi +94DkZfs0Nw4pgHBNrziGLp5/V6+eF67rHMsoIV+2HNjnogQi+dPa2MsCAwEAAaOB +sDCBrTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zArBgNVHRAEJDAi +gA8yMDA2MTEyNzIwMjM0MlqBDzIwMjYxMTI3MjA1MzQyWjAfBgNVHSMEGDAWgBRo +kORnpKZTgMeGZqTx90tD+4S9bTAdBgNVHQ4EFgQUaJDkZ6SmU4DHhmak8fdLQ/uE +vW0wHQYJKoZIhvZ9B0EABBAwDhsIVjcuMTo0LjADAgSQMA0GCSqGSIb3DQEBBQUA +A4IBAQCT1DCw1wMgKtD5Y+iRDAUgqV8ZyntyTtSx29CW+1RaGSwMCPeyvIWonX9t +O1KzKtvn1ISMY/YPyyYBkVBs9F8U4pN0wBOeMDpQ47RgxRzwIkSNcUesyBrJ6Zua +AGAT/3B+XxFNSRuzFVJ7yVTav52Vr2ua2J7p8eRDjeIRRDq/r72DQnNSi6q7pynP +9WQcCk3RvKqsnyrQ/39/2n3qse0wJcGE2jTSW3iDVuycNsMm4hH2Z0kdkquM++v/ +eu6FSqdQgPCnXEqULl8FmTxSQeDNtGPPAUO6nIPcj2A781q0tHuu2guQOHXvgR1m +0vdXcDazv/wor3ElhVsT/h5/WrQ8 +-----END CERTIFICATE----- + +# Operating CA: Entrust Datacard +# Issuer: CN=Entrust Root Certification Authority - EC1 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2012 Entrust, Inc. - for authorized use only +# Subject: CN=Entrust Root Certification Authority - EC1 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2012 Entrust, Inc. - for authorized use only +# Label: "Entrust Root Certification Authority - EC1" +# Serial: 51543124481930649114116133369 +# MD5 Fingerprint: b6:7e:1d:f0:58:c5:49:6c:24:3b:3d:ed:98:18:ed:bc +# SHA1 Fingerprint: 20:d8:06:40:df:9b:25:f5:12:25:3a:11:ea:f7:59:8a:eb:14:b5:47 +# SHA256 Fingerprint: 02:ed:0e:b2:8c:14:da:45:16:5c:56:67:91:70:0d:64:51:d7:fb:56:f0:b2:ab:1d:3b:8e:b0:70:e5:6e:df:f5 +-----BEGIN CERTIFICATE----- +MIIC+TCCAoCgAwIBAgINAKaLeSkAAAAAUNCR+TAKBggqhkjOPQQDAzCBvzELMAkG +A1UEBhMCVVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3 +d3cuZW50cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDEyIEVu +dHJ1c3QsIEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEzMDEGA1UEAxMq +RW50cnVzdCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRUMxMB4XDTEy +MTIxODE1MjUzNloXDTM3MTIxODE1NTUzNlowgb8xCzAJBgNVBAYTAlVTMRYwFAYD +VQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1c3QubmV0 +L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxMiBFbnRydXN0LCBJbmMuIC0g +Zm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxMzAxBgNVBAMTKkVudHJ1c3QgUm9vdCBD +ZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEVDMTB2MBAGByqGSM49AgEGBSuBBAAi +A2IABIQTydC6bUF74mzQ61VfZgIaJPRbiWlH47jCffHyAsWfoPZb1YsGGYZPUxBt +ByQnoaD41UcZYUx9ypMn6nQM72+WCf5j7HBdNq1nd67JnXxVRDqiY1Ef9eNi1KlH +Bz7MIKNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0O +BBYEFLdj5xrdjekIplWDpOBqUEFlEUJJMAoGCCqGSM49BAMDA2cAMGQCMGF52OVC +R98crlOZF7ZvHH3hvxGU0QOIdeSNiaSKd0bebWHvAvX7td/M/k7//qnmpwIwW5nX +hTcGtXsI/esni0qU+eH6p44mCOh8kmhtc9hvJqwhAriZtyZBWyVgrtBIGu4G +-----END CERTIFICATE----- + +# Operating CA: Entrust Datacard +# Issuer: CN=Entrust Root Certification Authority - G2 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2009 Entrust, Inc. - for authorized use only +# Subject: CN=Entrust Root Certification Authority - G2 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2009 Entrust, Inc. - for authorized use only +# Label: "Entrust Root Certification Authority - G2" +# Serial: 1246989352 +# MD5 Fingerprint: 4b:e2:c9:91:96:65:0c:f4:0e:5a:93:92:a0:0a:fe:b2 +# SHA1 Fingerprint: 8c:f4:27:fd:79:0c:3a:d1:66:06:8d:e8:1e:57:ef:bb:93:22:72:d4 +# SHA256 Fingerprint: 43:df:57:74:b0:3e:7f:ef:5f:e4:0d:93:1a:7b:ed:f1:bb:2e:6b:42:73:8c:4e:6d:38:41:10:3d:3a:a7:f3:39 +-----BEGIN CERTIFICATE----- +MIIEPjCCAyagAwIBAgIESlOMKDANBgkqhkiG9w0BAQsFADCBvjELMAkGA1UEBhMC +VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50 +cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3Qs +IEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVz +dCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzIwHhcNMDkwNzA3MTcy +NTU0WhcNMzAxMjA3MTc1NTU0WjCBvjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUVu +dHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVnYWwt +dGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0 +aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVzdCBSb290IENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5IC0gRzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC6hLZy254Ma+KZ6TABp3bqMriVQRrJ2mFOWHLP/vaCeb9zYQYKpSfYs1/T +RU4cctZOMvJyig/3gxnQaoCAAEUesMfnmr8SVycco2gvCoe9amsOXmXzHHfV1IWN +cCG0szLni6LVhjkCsbjSR87kyUnEO6fe+1R9V77w6G7CebI6C1XiUJgWMhNcL3hW +wcKUs/Ja5CeanyTXxuzQmyWC48zCxEXFjJd6BmsqEZ+pCm5IO2/b1BEZQvePB7/1 +U1+cPvQXLOZprE4yTGJ36rfo5bs0vBmLrpxR57d+tVOxMyLlbc9wPBr64ptntoP0 +jaWvYkxN4FisZDQSA/i2jZRjJKRxAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP +BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqciZ60B7vfec7aVHUbI2fkBJmqzAN +BgkqhkiG9w0BAQsFAAOCAQEAeZ8dlsa2eT8ijYfThwMEYGprmi5ZiXMRrEPR9RP/ +jTkrwPK9T3CMqS/qF8QLVJ7UG5aYMzyorWKiAHarWWluBh1+xLlEjZivEtRh2woZ +Rkfz6/djwUAFQKXSt/S1mja/qYh2iARVBCuch38aNzx+LaUa2NSJXsq9rD1s2G2v +1fN2D807iDginWyTmsQ9v4IbZT+mD12q/OWyFcq1rca8PdCE6OoGcrBNOTJ4vz4R +nAuknZoh8/CbCzB428Hch0P+vGOaysXCHMnHjf87ElgI5rY97HosTvuDls4MPGmH +VHOkc8KT/1EQrBVUAdj8BbGJoX90g5pJ19xOe4pIb4tF9g== +-----END CERTIFICATE----- + +# Operating CA: Entrust Datacard +# Issuer: CN=Entrust.net Certification Authority (2048) O=Entrust.net OU=www.entrust.net/CPS_2048 incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited +# Subject: CN=Entrust.net Certification Authority (2048) O=Entrust.net OU=www.entrust.net/CPS_2048 incorp. by ref. (limits liab.)/(c) 1999 Entrust.net Limited +# Label: "Entrust.net Premium 2048 Secure Server CA" +# Serial: 946069240 +# MD5 Fingerprint: ee:29:31:bc:32:7e:9a:e6:e8:b5:f7:51:b4:34:71:90 +# SHA1 Fingerprint: 50:30:06:09:1d:97:d4:f5:ae:39:f7:cb:e7:92:7d:7d:65:2d:34:31 +# SHA256 Fingerprint: 6d:c4:71:72:e0:1c:bc:b0:bf:62:58:0d:89:5f:e2:b8:ac:9a:d4:f8:73:80:1e:0c:10:b9:c8:37:d2:1e:b1:77 +-----BEGIN CERTIFICATE----- +MIIEKjCCAxKgAwIBAgIEOGPe+DANBgkqhkiG9w0BAQUFADCBtDEUMBIGA1UEChML +RW50cnVzdC5uZXQxQDA+BgNVBAsUN3d3dy5lbnRydXN0Lm5ldC9DUFNfMjA0OCBp +bmNvcnAuIGJ5IHJlZi4gKGxpbWl0cyBsaWFiLikxJTAjBgNVBAsTHChjKSAxOTk5 +IEVudHJ1c3QubmV0IExpbWl0ZWQxMzAxBgNVBAMTKkVudHJ1c3QubmV0IENlcnRp +ZmljYXRpb24gQXV0aG9yaXR5ICgyMDQ4KTAeFw05OTEyMjQxNzUwNTFaFw0yOTA3 +MjQxNDE1MTJaMIG0MRQwEgYDVQQKEwtFbnRydXN0Lm5ldDFAMD4GA1UECxQ3d3d3 +LmVudHJ1c3QubmV0L0NQU18yMDQ4IGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxp +YWIuKTElMCMGA1UECxMcKGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDEzMDEG +A1UEAxMqRW50cnVzdC5uZXQgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgKDIwNDgp +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArU1LqRKGsuqjIAcVFmQq +K0vRvwtKTY7tgHalZ7d4QMBzQshowNtTK91euHaYNZOLGp18EzoOH1u3Hs/lJBQe +sYGpjX24zGtLA/ECDNyrpUAkAH90lKGdCCmziAv1h3edVc3kw37XamSrhRSGlVuX +MlBvPci6Zgzj/L24ScF2iUkZ/cCovYmjZy/Gn7xxGWC4LeksyZB2ZnuU4q941mVT +XTzWnLLPKQP5L6RQstRIzgUyVYr9smRMDuSYB3Xbf9+5CFVghTAp+XtIpGmG4zU/ +HoZdenoVve8AjhUiVBcAkCaTvA5JaJG/+EfTnZVCwQ5N328mz8MYIWJmQ3DW1cAH +4QIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUVeSB0RGAvtiJuQijMfmhJAkWuXAwDQYJKoZIhvcNAQEFBQADggEBADub +j1abMOdTmXx6eadNl9cZlZD7Bh/KM3xGY4+WZiT6QBshJ8rmcnPyT/4xmf3IDExo +U8aAghOY+rat2l098c5u9hURlIIM7j+VrxGrD9cv3h8Dj1csHsm7mhpElesYT6Yf +zX1XEC+bBAlahLVu2B064dae0Wx5XnkcFMXj0EyTO2U87d89vqbllRrDtRnDvV5b +u/8j72gZyxKTJ1wDLW8w0B62GqzeWvfRqqgnpv55gcR5mTNXuhKwqeBCbJPKVt7+ +bYQLCIt+jerXmCHG8+c8eS9enNFMFY3h7CI3zJpDC5fcgJCNs2ebb0gIFVbPv/Er +fF6adulZkMV8gzURZVE= +-----END CERTIFICATE----- + +# Operating CA: Entrust Datacard +# Issuer: CN=AffirmTrust Commercial O=AffirmTrust +# Subject: CN=AffirmTrust Commercial O=AffirmTrust +# Label: "AffirmTrust Commercial" +# Serial: 8608355977964138876 +# MD5 Fingerprint: 82:92:ba:5b:ef:cd:8a:6f:a6:3d:55:f9:84:f6:d6:b7 +# SHA1 Fingerprint: f9:b5:b6:32:45:5f:9c:be:ec:57:5f:80:dc:e9:6e:2c:c7:b2:78:b7 +# SHA256 Fingerprint: 03:76:ab:1d:54:c5:f9:80:3c:e4:b2:e2:01:a0:ee:7e:ef:7b:57:b6:36:e8:a9:3c:9b:8d:48:60:c9:6f:5f:a7 +-----BEGIN CERTIFICATE----- +MIIDTDCCAjSgAwIBAgIId3cGJyapsXwwDQYJKoZIhvcNAQELBQAwRDELMAkGA1UE +BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZpcm1UcnVz +dCBDb21tZXJjaWFsMB4XDTEwMDEyOTE0MDYwNloXDTMwMTIzMTE0MDYwNlowRDEL +MAkGA1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZp +cm1UcnVzdCBDb21tZXJjaWFsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEA9htPZwcroRX1BiLLHwGy43NFBkRJLLtJJRTWzsO3qyxPxkEylFf6EqdbDuKP +Hx6GGaeqtS25Xw2Kwq+FNXkyLbscYjfysVtKPcrNcV/pQr6U6Mje+SJIZMblq8Yr +ba0F8PrVC8+a5fBQpIs7R6UjW3p6+DM/uO+Zl+MgwdYoic+U+7lF7eNAFxHUdPAL +MeIrJmqbTFeurCA+ukV6BfO9m2kVrn1OIGPENXY6BwLJN/3HR+7o8XYdcxXyl6S1 +yHp52UKqK39c/s4mT6NmgTWvRLpUHhwwMmWd5jyTXlBOeuM61G7MGvv50jeuJCqr +VwMiKA1JdX+3KNp1v47j3A55MQIDAQABo0IwQDAdBgNVHQ4EFgQUnZPGU4teyq8/ +nx4P5ZmVvCT2lI8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJ +KoZIhvcNAQELBQADggEBAFis9AQOzcAN/wr91LoWXym9e2iZWEnStB03TX8nfUYG +XUPGhi4+c7ImfU+TqbbEKpqrIZcUsd6M06uJFdhrJNTxFq7YpFzUf1GO7RgBsZNj +vbz4YYCanrHOQnDiqX0GJX0nof5v7LMeJNrjS1UaADs1tDvZ110w/YETifLCBivt +Z8SOyUOyXGsViQK8YvxO8rUzqrJv0wqiUOP2O+guRMLbZjipM1ZI8W0bM40NjD9g +N53Tym1+NH4Nn3J2ixufcv1SNUFFApYvHLKac0khsUlHRUe072o0EclNmsxZt9YC +nlpOZbWUrhvfKbAW8b8Angc6F2S1BLUjIZkKlTuXfO8= +-----END CERTIFICATE----- + +# Operating CA: Entrust Datacard +# Issuer: CN=AffirmTrust Networking O=AffirmTrust +# Subject: CN=AffirmTrust Networking O=AffirmTrust +# Label: "AffirmTrust Networking" +# Serial: 8957382827206547757 +# MD5 Fingerprint: 42:65:ca:be:01:9a:9a:4c:a9:8c:41:49:cd:c0:d5:7f +# SHA1 Fingerprint: 29:36:21:02:8b:20:ed:02:f5:66:c5:32:d1:d6:ed:90:9f:45:00:2f +# SHA256 Fingerprint: 0a:81:ec:5a:92:97:77:f1:45:90:4a:f3:8d:5d:50:9f:66:b5:e2:c5:8f:cd:b5:31:05:8b:0e:17:f3:f0:b4:1b +-----BEGIN CERTIFICATE----- +MIIDTDCCAjSgAwIBAgIIfE8EORzUmS0wDQYJKoZIhvcNAQEFBQAwRDELMAkGA1UE +BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZpcm1UcnVz +dCBOZXR3b3JraW5nMB4XDTEwMDEyOTE0MDgyNFoXDTMwMTIzMTE0MDgyNFowRDEL +MAkGA1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZp +cm1UcnVzdCBOZXR3b3JraW5nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEAtITMMxcua5Rsa2FSoOujz3mUTOWUgJnLVWREZY9nZOIG41w3SfYvm4SEHi3y +YJ0wTsyEheIszx6e/jarM3c1RNg1lho9Nuh6DtjVR6FqaYvZ/Ls6rnla1fTWcbua +kCNrmreIdIcMHl+5ni36q1Mr3Lt2PpNMCAiMHqIjHNRqrSK6mQEubWXLviRmVSRL +QESxG9fhwoXA3hA/Pe24/PHxI1Pcv2WXb9n5QHGNfb2V1M6+oF4nI979ptAmDgAp +6zxG8D1gvz9Q0twmQVGeFDdCBKNwV6gbh+0t+nvujArjqWaJGctB+d1ENmHP4ndG +yH329JKBNv3bNPFyfvMMFr20FQIDAQABo0IwQDAdBgNVHQ4EFgQUBx/S55zawm6i +QLSwelAQUHTEyL0wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJ +KoZIhvcNAQEFBQADggEBAIlXshZ6qML91tmbmzTCnLQyFE2npN/svqe++EPbkTfO +tDIuUFUaNU52Q3Eg75N3ThVwLofDwR1t3Mu1J9QsVtFSUzpE0nPIxBsFZVpikpzu +QY0x2+c06lkh1QF612S4ZDnNye2v7UsDSKegmQGA3GWjNq5lWUhPgkvIZfFXHeVZ +Lgo/bNjR9eUJtGxUAArgFU2HdW23WJZa3W3SAKD0m0i+wzekujbgfIeFlxoVot4u +olu9rxj5kFDNcFn4J2dHy8egBzp90SxdbBk6ZrV9/ZFvgrG+CJPbFEfxojfHRZ48 +x3evZKiT3/Zpg4Jg8klCNO1aAFSFHBY2kgxc+qatv9s= +-----END CERTIFICATE----- + +# Operating CA: Entrust Datacard +# Issuer: CN=AffirmTrust Premium O=AffirmTrust +# Subject: CN=AffirmTrust Premium O=AffirmTrust +# Label: "AffirmTrust Premium" +# Serial: 7893706540734352110 +# MD5 Fingerprint: c4:5d:0e:48:b6:ac:28:30:4e:0a:bc:f9:38:16:87:57 +# SHA1 Fingerprint: d8:a6:33:2c:e0:03:6f:b1:85:f6:63:4f:7d:6a:06:65:26:32:28:27 +# SHA256 Fingerprint: 70:a7:3f:7f:37:6b:60:07:42:48:90:45:34:b1:14:82:d5:bf:0e:69:8e:cc:49:8d:f5:25:77:eb:f2:e9:3b:9a +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIIbYwURrGmCu4wDQYJKoZIhvcNAQEMBQAwQTELMAkGA1UE +BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MRwwGgYDVQQDDBNBZmZpcm1UcnVz +dCBQcmVtaXVtMB4XDTEwMDEyOTE0MTAzNloXDTQwMTIzMTE0MTAzNlowQTELMAkG +A1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MRwwGgYDVQQDDBNBZmZpcm1U +cnVzdCBQcmVtaXVtMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxBLf +qV/+Qd3d9Z+K4/as4Tx4mrzY8H96oDMq3I0gW64tb+eT2TZwamjPjlGjhVtnBKAQ +JG9dKILBl1fYSCkTtuG+kU3fhQxTGJoeJKJPj/CihQvL9Cl/0qRY7iZNyaqoe5rZ ++jjeRFcV5fiMyNlI4g0WJx0eyIOFJbe6qlVBzAMiSy2RjYvmia9mx+n/K+k8rNrS +s8PhaJyJ+HoAVt70VZVs+7pk3WKL3wt3MutizCaam7uqYoNMtAZ6MMgpv+0GTZe5 +HMQxK9VfvFMSF5yZVylmd2EhMQcuJUmdGPLu8ytxjLW6OQdJd/zvLpKQBY0tL3d7 +70O/Nbua2Plzpyzy0FfuKE4mX4+QaAkvuPjcBukumj5Rp9EixAqnOEhss/n/fauG +V+O61oV4d7pD6kh/9ti+I20ev9E2bFhc8e6kGVQa9QPSdubhjL08s9NIS+LI+H+S +qHZGnEJlPqQewQcDWkYtuJfzt9WyVSHvutxMAJf7FJUnM7/oQ0dG0giZFmA7mn7S +5u046uwBHjxIVkkJx0w3AJ6IDsBz4W9m6XJHMD4Q5QsDyZpCAGzFlH5hxIrff4Ia +C1nEWTJ3s7xgaVY5/bQGeyzWZDbZvUjthB9+pSKPKrhC9IK31FOQeE4tGv2Bb0TX +OwF0lkLgAOIua+rF7nKsu7/+6qqo+Nz2snmKtmcCAwEAAaNCMEAwHQYDVR0OBBYE +FJ3AZ6YMItkm9UWrpmVSESfYRaxjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ +BAQDAgEGMA0GCSqGSIb3DQEBDAUAA4ICAQCzV00QYk465KzquByvMiPIs0laUZx2 +KI15qldGF9X1Uva3ROgIRL8YhNILgM3FEv0AVQVhh0HctSSePMTYyPtwni94loMg +Nt58D2kTiKV1NpgIpsbfrM7jWNa3Pt668+s0QNiigfV4Py/VpfzZotReBA4Xrf5B +8OWycvpEgjNC6C1Y91aMYj+6QrCcDFx+LmUmXFNPALJ4fqENmS2NuB2OosSw/WDQ +MKSOyARiqcTtNd56l+0OOF6SL5Nwpamcb6d9Ex1+xghIsV5n61EIJenmJWtSKZGc +0jlzCFfemQa0W50QBuHCAKi4HEoCChTQwUHK+4w1IX2COPKpVJEZNZOUbWo6xbLQ +u4mGk+ibyQ86p3q4ofB4Rvr8Ny/lioTz3/4E2aFooC8k4gmVBtWVyuEklut89pMF +u+1z6S3RdTnX5yTb2E5fQ4+e0BQ5v1VwSJlXMbSc7kqYA5YwH2AG7hsj/oFgIxpH +YoWlzBk0gG+zrBrjn/B7SK3VAdlntqlyk+otZrWyuOQ9PLLvTIzq6we/qzWaVYa8 +GKa1qF60g2xraUDTn9zxw2lrueFtCfTxqlB2Cnp9ehehVZZCmTEJ3WARjQUwfuaO +RtGdFNrHF+QFlozEJLUbzxQHskD4o55BhrwE0GuWyCqANP2/7waj3VjFhT0+j/6e +KeC2uAloGRwYQw== +-----END CERTIFICATE----- + +# Operating CA: Entrust Datacard +# Issuer: CN=AffirmTrust Premium ECC O=AffirmTrust +# Subject: CN=AffirmTrust Premium ECC O=AffirmTrust +# Label: "AffirmTrust Premium ECC" +# Serial: 8401224907861490260 +# MD5 Fingerprint: 64:b0:09:55:cf:b1:d5:99:e2:be:13:ab:a6:5d:ea:4d +# SHA1 Fingerprint: b8:23:6b:00:2f:1d:16:86:53:01:55:6c:11:a4:37:ca:eb:ff:c3:bb +# SHA256 Fingerprint: bd:71:fd:f6:da:97:e4:cf:62:d1:64:7a:dd:25:81:b0:7d:79:ad:f8:39:7e:b4:ec:ba:9c:5e:84:88:82:14:23 +-----BEGIN CERTIFICATE----- +MIIB/jCCAYWgAwIBAgIIdJclisc/elQwCgYIKoZIzj0EAwMwRTELMAkGA1UEBhMC +VVMxFDASBgNVBAoMC0FmZmlybVRydXN0MSAwHgYDVQQDDBdBZmZpcm1UcnVzdCBQ +cmVtaXVtIEVDQzAeFw0xMDAxMjkxNDIwMjRaFw00MDEyMzExNDIwMjRaMEUxCzAJ +BgNVBAYTAlVTMRQwEgYDVQQKDAtBZmZpcm1UcnVzdDEgMB4GA1UEAwwXQWZmaXJt +VHJ1c3QgUHJlbWl1bSBFQ0MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQNMF4bFZ0D +0KF5Nbc6PJJ6yhUczWLznCZcBz3lVPqj1swS6vQUX+iOGasvLkjmrBhDeKzQN8O9 +ss0s5kfiGuZjuD0uL3jET9v0D6RoTFVya5UdThhClXjMNzyR4ptlKymjQjBAMB0G +A1UdDgQWBBSaryl6wBE1NSZRMADDav5A1a7WPDAPBgNVHRMBAf8EBTADAQH/MA4G +A1UdDwEB/wQEAwIBBjAKBggqhkjOPQQDAwNnADBkAjAXCfOHiFBar8jAQr9HX/Vs +aobgxCd05DhT1wV/GzTjxi+zygk8N53X57hG8f2h4nECMEJZh0PUUd+60wkyWs6I +flc9nF9Ca/UHLbXwgpP5WW+uZPpY5Yse42O+tYHNbwKMeQ== +-----END CERTIFICATE----- + +# Operating CA: GlobalSign +# Issuer: CN=GlobalSign Root CA O=GlobalSign nv-sa OU=Root CA +# Subject: CN=GlobalSign Root CA O=GlobalSign nv-sa OU=Root CA +# Label: "GlobalSign Root CA" +# Serial: 4835703278459707669005204 +# MD5 Fingerprint: 3e:45:52:15:09:51:92:e1:b7:5d:37:9f:b1:87:29:8a +# SHA1 Fingerprint: b1:bc:96:8b:d4:f4:9d:62:2a:a8:9a:81:f2:15:01:52:a4:1d:82:9c +# SHA256 Fingerprint: eb:d4:10:40:e4:bb:3e:c7:42:c9:e3:81:d3:1e:f2:a4:1a:48:b6:68:5c:96:e7:ce:f3:c1:df:6c:d4:33:1c:99 +-----BEGIN CERTIFICATE----- +MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG +A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv +b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw +MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i +YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT +aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ +jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp +xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp +1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG +snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ +U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8 +9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E +BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B +AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz +yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE +38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP +AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad +DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME +HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== +-----END CERTIFICATE----- + +# Operating CA: GlobalSign +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R3 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R3 +# Label: "GlobalSign Root CA - R3" +# Serial: 4835703278459759426209954 +# MD5 Fingerprint: c5:df:b8:49:ca:05:13:55:ee:2d:ba:1a:c3:3e:b0:28 +# SHA1 Fingerprint: d6:9b:56:11:48:f0:1c:77:c5:45:78:c1:09:26:df:5b:85:69:76:ad +# SHA256 Fingerprint: cb:b5:22:d7:b7:f1:27:ad:6a:01:13:86:5b:df:1c:d4:10:2e:7d:07:59:af:63:5a:7c:f4:72:0d:c9:63:c5:3b +-----BEGIN CERTIFICATE----- +MIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G +A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp +Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4 +MTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG +A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8 +RgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT +gHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm +KPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd +QQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ +XriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw +DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o +LkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU +RUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp +jjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK +6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX +mcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs +Mx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH +WD9f +-----END CERTIFICATE----- + +# Operating CA: GlobalSign +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R5 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R5 +# Label: "GlobalSign ECC Root CA - R5" +# Serial: 32785792099990507226680698011560947931244 +# MD5 Fingerprint: 9f:ad:3b:1c:02:1e:8a:ba:17:74:38:81:0c:a2:bc:08 +# SHA1 Fingerprint: 1f:24:c6:30:cd:a4:18:ef:20:69:ff:ad:4f:dd:5f:46:3a:1b:69:aa +# SHA256 Fingerprint: 17:9f:bc:14:8a:3d:d0:0f:d2:4e:a1:34:58:cc:43:bf:a7:f5:9c:81:82:d7:83:a5:13:f6:eb:ec:10:0c:89:24 +-----BEGIN CERTIFICATE----- +MIICHjCCAaSgAwIBAgIRYFlJ4CYuu1X5CneKcflK2GwwCgYIKoZIzj0EAwMwUDEk +MCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI1MRMwEQYDVQQKEwpH +bG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoX +DTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBD +QSAtIFI1MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu +MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAER0UOlvt9Xb/pOdEh+J8LttV7HpI6SFkc +8GIxLcB6KP4ap1yztsyX50XUWPrRd21DosCHZTQKH3rd6zwzocWdTaRvQZU4f8ke +hOvRnkmSh5SHDDqFSmafnVmTTZdhBoZKo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYD +VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUPeYpSJvqB8ohREom3m7e0oPQn1kwCgYI +KoZIzj0EAwMDaAAwZQIxAOVpEslu28YxuglB4Zf4+/2a4n0Sye18ZNPLBSWLVtmg +515dTguDnFt2KaAJJiFqYgIwcdK1j1zqO+F4CYWodZI7yFz9SO8NdCKoCOJuxUnO +xwy8p2Fp8fc74SrL+SvzZpA3 +-----END CERTIFICATE----- + +# Operating CA: GlobalSign +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Label: "GlobalSign Root CA - R6" +# Serial: 1417766617973444989252670301619537 +# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae +# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1 +# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69 +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg +MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx +MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET +MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI +xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k +ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD +aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw +LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw +1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX +k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 +SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h +bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n +WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY +rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce +MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu +bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN +nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt +Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 +55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj +vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf +cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz +oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp +nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs +pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v +JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R +8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 +5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= +-----END CERTIFICATE----- + +# Note: "GlobalSign Root CA - R7" not added on purpose. It is P-521. + +# Operating CA: GoDaddy +# Issuer: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc. +# Subject: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc. +# Label: "Go Daddy Root Certificate Authority - G2" +# Serial: 0 +# MD5 Fingerprint: 80:3a:bc:22:c1:e6:fb:8d:9b:3b:27:4a:32:1b:9a:01 +# SHA1 Fingerprint: 47:be:ab:c9:22:ea:e8:0e:78:78:34:62:a7:9f:45:c2:54:fd:e6:8b +# SHA256 Fingerprint: 45:14:0b:32:47:eb:9c:c8:c5:b4:f0:d7:b5:30:91:f7:32:92:08:9e:6e:5a:63:e2:74:9d:d3:ac:a9:19:8e:da +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT +EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp +ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAwMFoXDTM3MTIzMTIz +NTk1OVowgYMxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH +EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjExMC8GA1UE +AxMoR28gRGFkZHkgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9xYgjx+lk09xvJGKP3gElY6SKD +E6bFIEMBO4Tx5oVJnyfq9oQbTqC023CYxzIBsQU+B07u9PpPL1kwIuerGVZr4oAH +/PMWdYA5UXvl+TW2dE6pjYIT5LY/qQOD+qK+ihVqf94Lw7YZFAXK6sOoBJQ7Rnwy +DfMAZiLIjWltNowRGLfTshxgtDj6AozO091GB94KPutdfMh8+7ArU6SSYmlRJQVh +GkSBjCypQ5Yj36w6gZoOKcUcqeldHraenjAKOc7xiID7S13MMuyFYkMlNAJWJwGR +tDtwKj9useiciAF9n9T521NtYJ2/LOdYq7hfRvzOxBsDPAnrSTFcaUaz4EcCAwEA +AaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FDqahQcQZyi27/a9BUFuIMGU2g/eMA0GCSqGSIb3DQEBCwUAA4IBAQCZ21151fmX +WWcDYfF+OwYxdS2hII5PZYe096acvNjpL9DbWu7PdIxztDhC2gV7+AJ1uP2lsdeu +9tfeE8tTEH6KRtGX+rcuKxGrkLAngPnon1rpN5+r5N9ss4UXnT3ZJE95kTXWXwTr +gIOrmgIttRD02JDHBHNA7XIloKmf7J6raBKZV8aPEjoJpL1E/QYVN8Gb5DKj7Tjo +2GTzLH4U/ALqn83/B2gX2yKQOC16jdFU8WnjXzPKej17CuPKf1855eJ1usV2GDPO +LPAvTK33sefOT6jEm0pUBsV/fdUID+Ic/n4XuKxe9tQWskMJDE32p2u0mYRlynqI +4uJEvlz36hz1 +-----END CERTIFICATE----- + +# Operating CA: GoDaddy +# Issuer: CN=Starfield Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Subject: CN=Starfield Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Label: "Starfield Root Certificate Authority - G2" +# Serial: 0 +# MD5 Fingerprint: d6:39:81:c6:52:7e:96:69:fc:fc:ca:66:ed:05:f2:96 +# SHA1 Fingerprint: b5:1c:06:7c:ee:2b:0c:3d:f8:55:ab:2d:92:f4:fe:39:d4:e7:0f:0e +# SHA256 Fingerprint: 2c:e1:cb:0b:f9:d2:f9:e1:02:99:3f:be:21:51:52:c3:b2:dd:0c:ab:de:1c:68:e5:31:9b:83:91:54:db:b7:f5 +-----BEGIN CERTIFICATE----- +MIID3TCCAsWgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBjzELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT +HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAMTKVN0YXJmaWVs +ZCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAw +MFoXDTM3MTIzMTIzNTk1OVowgY8xCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6 +b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFyZmllbGQgVGVj +aG5vbG9naWVzLCBJbmMuMTIwMAYDVQQDEylTdGFyZmllbGQgUm9vdCBDZXJ0aWZp +Y2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAL3twQP89o/8ArFvW59I2Z154qK3A2FWGMNHttfKPTUuiUP3oWmb3ooa/RMg +nLRJdzIpVv257IzdIvpy3Cdhl+72WoTsbhm5iSzchFvVdPtrX8WJpRBSiUZV9Lh1 +HOZ/5FSuS/hVclcCGfgXcVnrHigHdMWdSL5stPSksPNkN3mSwOxGXn/hbVNMYq/N +Hwtjuzqd+/x5AJhhdM8mgkBj87JyahkNmcrUDnXMN/uLicFZ8WJ/X7NfZTD4p7dN +dloedl40wOiWVpmKs/B/pM293DIxfJHP4F8R+GuqSVzRmZTRouNjWwl2tVZi4Ut0 +HZbUJtQIBFnQmA4O5t78w+wfkPECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFHwMMh+n2TB/xH1oo2Kooc6rB1snMA0G +CSqGSIb3DQEBCwUAA4IBAQARWfolTwNvlJk7mh+ChTnUdgWUXuEok21iXQnCoKjU +sHU48TRqneSfioYmUeYs0cYtbpUgSpIB7LiKZ3sx4mcujJUDJi5DnUox9g61DLu3 +4jd/IroAow57UvtruzvE03lRTs2Q9GcHGcg8RnoNAX3FWOdt5oUwF5okxBDgBPfg +8n/Uqgr/Qh037ZTlZFkSIHc40zI+OIF1lnP6aI+xy84fxez6nH7PfrHxBy22/L/K +pL/QlwVKvOoYKAKQvVR4CSFx09F9HdkWsKlhPdAKACL8x3vLCWRFCztAgfd9fDL1 +mMpYjn0q7pBZc2T5NnReJaH1ZgUufzkVqSr7UIuOhWn0 +-----END CERTIFICATE----- + +# Operating CA: GoDaddy +# Issuer: O=Starfield Technologies, Inc. OU=Starfield Class 2 Certification Authority +# Subject: O=Starfield Technologies, Inc. OU=Starfield Class 2 Certification Authority +# Label: "Starfield Class 2 CA" +# Serial: 0 +# MD5 Fingerprint: 32:4a:4b:bb:c8:63:69:9b:be:74:9a:c6:dd:1d:46:24 +# SHA1 Fingerprint: ad:7e:1c:28:b0:64:ef:8f:60:03:40:20:14:c3:d0:e3:37:0e:b5:8a +# SHA256 Fingerprint: 14:65:fa:20:53:97:b8:76:fa:a6:f0:a9:95:8e:55:90:e4:0f:cc:7f:aa:4f:b7:c2:c8:67:75:21:fb:5f:b6:58 +-----BEGIN CERTIFICATE----- +MIIEDzCCAvegAwIBAgIBADANBgkqhkiG9w0BAQUFADBoMQswCQYDVQQGEwJVUzEl +MCMGA1UEChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMp +U3RhcmZpZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQw +NjI5MTczOTE2WhcNMzQwNjI5MTczOTE2WjBoMQswCQYDVQQGEwJVUzElMCMGA1UE +ChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMpU3RhcmZp +ZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggEgMA0GCSqGSIb3 +DQEBAQUAA4IBDQAwggEIAoIBAQC3Msj+6XGmBIWtDBFk385N78gDGIc/oav7PKaf +8MOh2tTYbitTkPskpD6E8J7oX+zlJ0T1KKY/e97gKvDIr1MvnsoFAZMej2YcOadN ++lq2cwQlZut3f+dZxkqZJRRU6ybH838Z1TBwj6+wRir/resp7defqgSHo9T5iaU0 +X9tDkYI22WY8sbi5gv2cOj4QyDvvBmVmepsZGD3/cVE8MC5fvj13c7JdBmzDI1aa +K4UmkhynArPkPw2vCHmCuDY96pzTNbO8acr1zJ3o/WSNF4Azbl5KXZnJHoe0nRrA +1W4TNSNe35tfPe/W93bC6j67eA0cQmdrBNj41tpvi/JEoAGrAgEDo4HFMIHCMB0G +A1UdDgQWBBS/X7fRzt0fhvRbVazc1xDCDqmI5zCBkgYDVR0jBIGKMIGHgBS/X7fR +zt0fhvRbVazc1xDCDqmI56FspGowaDELMAkGA1UEBhMCVVMxJTAjBgNVBAoTHFN0 +YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAsTKVN0YXJmaWVsZCBD +bGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQEFBQADggEBAAWdP4id0ckaVaGsafPzWdqbAYcaT1epoXkJKtv3 +L7IezMdeatiDh6GX70k1PncGQVhiv45YuApnP+yz3SFmH8lU+nLMPUxA2IGvd56D +eruix/U0F47ZEUD0/CwqTRV/p2JdLiXTAAsgGh1o+Re49L2L7ShZ3U0WixeDyLJl +xy16paq8U4Zt3VekyvggQQto8PT7dL5WXXp59fkdheMtlb71cZBDzI0fmgAKhynp +VSJYACPq4xJDKVtHCN2MQWplBqjlIapBtJUhlbl90TSrE9atvNziPTnNvT51cKEY +WQPJIrSPnNVeKtelttQKbfi3QBFGmh95DmK/D5fs4C8fF5Q= +-----END CERTIFICATE----- + +# Operating CA: GoDaddy +# Issuer: O=The Go Daddy Group, Inc. OU=Go Daddy Class 2 Certification Authority +# Subject: O=The Go Daddy Group, Inc. OU=Go Daddy Class 2 Certification Authority +# Label: "Go Daddy Class 2 CA" +# Serial: 0 +# MD5 Fingerprint: 91:de:06:25:ab:da:fd:32:17:0c:bb:25:17:2a:84:67 +# SHA1 Fingerprint: 27:96:ba:e6:3f:18:01:e2:77:26:1b:a0:d7:77:70:02:8f:20:ee:e4 +# SHA256 Fingerprint: c3:84:6b:f2:4b:9e:93:ca:64:27:4c:0e:c6:7c:1e:cc:5e:02:4f:fc:ac:d2:d7:40:19:35:0e:81:fe:54:6a:e4 +-----BEGIN CERTIFICATE----- +MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEh +MB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBE +YWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3 +MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRo +ZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3Mg +MiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggEN +ADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCA +PVYYYwhv2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6w +wdhFJ2+qN1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXi +EqITLdiOr18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMY +avx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+ +YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0OBBYEFNLE +sNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h +/t2oatTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5 +IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD +ggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wimPQoZ+YeAEW5p5JYXMP80kWNy +OO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKtI3lpjbi2Tc7P +TMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQ +HmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mER +dEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5Cuf +ReYNnyicsbkqWletNw+vHX/bvZ8= +-----END CERTIFICATE----- + +# Operating CA: Google Trust Services LLC +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R2 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R2 +# Label: "GlobalSign Root CA - R2" +# Serial: 4835703278459682885658125 +# MD5 Fingerprint: 94:14:77:7e:3e:5e:fd:8f:30:bd:41:b0:cf:e7:d0:30 +# SHA1 Fingerprint: 75:e0:ab:b6:13:85:12:27:1c:04:f8:5f:dd:de:38:e4:b7:24:2e:fe +# SHA256 Fingerprint: ca:42:dd:41:74:5f:d0:b8:1e:b9:02:36:2c:f9:d8:bf:71:9d:a1:bd:1b:1e:fc:94:6f:5b:4c:99:f4:2c:1b:9e +-----BEGIN CERTIFICATE----- +MIIDujCCAqKgAwIBAgILBAAAAAABD4Ym5g0wDQYJKoZIhvcNAQEFBQAwTDEgMB4G +A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjIxEzARBgNVBAoTCkdsb2JhbFNp +Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDYxMjE1MDgwMDAwWhcNMjExMjE1 +MDgwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEG +A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAKbPJA6+Lm8omUVCxKs+IVSbC9N/hHD6ErPL +v4dfxn+G07IwXNb9rfF73OX4YJYJkhD10FPe+3t+c4isUoh7SqbKSaZeqKeMWhG8 +eoLrvozps6yWJQeXSpkqBy+0Hne/ig+1AnwblrjFuTosvNYSuetZfeLQBoZfXklq +tTleiDTsvHgMCJiEbKjNS7SgfQx5TfC4LcshytVsW33hoCmEofnTlEnLJGKRILzd +C9XZzPnqJworc5HGnRusyMvo4KD0L5CLTfuwNhv2GXqF4G3yYROIXJ/gkwpRl4pa +zq+r1feqCapgvdzZX99yqWATXgAByUr6P6TqBwMhAo6CygPCm48CAwEAAaOBnDCB +mTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUm+IH +V2ccHsBqBt5ZtJot39wZhi4wNgYDVR0fBC8wLTAroCmgJ4YlaHR0cDovL2NybC5n +bG9iYWxzaWduLm5ldC9yb290LXIyLmNybDAfBgNVHSMEGDAWgBSb4gdXZxwewGoG +3lm0mi3f3BmGLjANBgkqhkiG9w0BAQUFAAOCAQEAmYFThxxol4aR7OBKuEQLq4Gs +J0/WwbgcQ3izDJr86iw8bmEbTUsp9Z8FHSbBuOmDAGJFtqkIk7mpM0sYmsL4h4hO +291xNBrBVNpGP+DTKqttVCL1OmLNIG+6KYnX3ZHu01yiPqFbQfXf5WRDLenVOavS +ot+3i9DAgBkcRcAtjOj4LaR0VknFBbVPFd5uRHg5h6h+u/N5GJG79G+dwfCMNYxd +AfvDbbnvRG15RjF+Cv6pgsH/76tuIMRQyV+dTZsXjAzlAcmgQWpzU/qlULRuJQ/7 +TBj0/VLZjmmx6BEP3ojY+x1J96relc8geMJgEtslQIxq/H5COEBkEveegeGTLg== +-----END CERTIFICATE----- + +# Operating CA: Google Trust Services LLC +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R4 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R4 +# Label: "GlobalSign ECC Root CA - R4" +# Serial: 14367148294922964480859022125800977897474 +# MD5 Fingerprint: 20:f0:27:68:d1:7e:a0:9d:0e:e6:2a:ca:df:5c:89:8e +# SHA1 Fingerprint: 69:69:56:2e:40:80:f4:24:a1:e7:19:9f:14:ba:f3:ee:58:ab:6a:bb +# SHA256 Fingerprint: be:c9:49:11:c2:95:56:76:db:6c:0a:55:09:86:d7:6e:3b:a0:05:66:7c:44:2c:97:62:b4:fb:b7:73:de:22:8c +-----BEGIN CERTIFICATE----- +MIIB4TCCAYegAwIBAgIRKjikHJYKBN5CsiilC+g0mAIwCgYIKoZIzj0EAwIwUDEk +MCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI0MRMwEQYDVQQKEwpH +bG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoX +DTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBD +QSAtIFI0MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuMZ5049sJQ6fLjkZHAOkrprlOQcJ +FspjsbmG+IpXwVfOQvpzofdlQv8ewQCybnMO/8ch5RikqtlxP6jUuc6MHaNCMEAw +DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFFSwe61F +uOJAf/sKbvu+M8k8o4TVMAoGCCqGSM49BAMCA0gAMEUCIQDckqGgE6bPA7DmxCGX +kPoUVy0D7O48027KqGx2vKLeuwIgJ6iFJzWbVsaj8kfSt24bAgAXqmemFZHe+pTs +ewv4n4Q= +-----END CERTIFICATE----- + +# Operating CA: Google Trust Services LLC +# Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R1 +# Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R1 +# Label: "GTS Root R1" +# Serial: 6e:47:a9:c5:4b:47:0c:0d:ec:33:d0:89:b9:1c:f4:e1 +# MD5 Fingerprint: 82:1A:EF:D4:D2:4A:F2:9F:E2:3D:97:06:14:70:72:85 +# SHA1 Fingerprint: E1:C9:50:E6:EF:22:F8:4C:56:45:72:8B:92:20:60:D7:D5:A7:A3:E8 +# SHA256 Fingerprint: 2A:57:54:71:E3:13:40:BC:21:58:1C:BD:2C:F1:3E:15:84:63:20:3E:CE:94:BC:F9:D3:CC:19:6B:F0:9A:54:72 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxUtHDA3sM9CJuRz04TANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaM +f/vo27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vX +mX7wCl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7 +zUjwTcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0P +fyblqAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtc +vfaHszVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4 +Zor8Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUsp +zBmkMiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOO +Rc92wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYW +k70paDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+ +DVrNVjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgF +lQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBADiW +Cu49tJYeX++dnAsznyvgyv3SjgofQXSlfKqE1OXyHuY3UjKcC9FhHb8owbZEKTV1 +d5iyfNm9dKyKaOOpMQkpAWBz40d8U6iQSifvS9efk+eCNs6aaAyC58/UEBZvXw6Z +XPYfcX3v73svfuo21pdwCxXu11xWajOl40k4DLh9+42FpLFZXvRq4d2h9mREruZR +gyFmxhE+885H7pwoHyXa/6xmld01D1zvICxi/ZG6qcz8WpyTgYMpl0p8WnK0OdC3 +d8t5/Wk6kjftbjhlRn7pYL15iJdfOBL07q9bgsiG1eGZbYwE8na6SfZu6W0eX6Dv +J4J2QPim01hcDyxC2kLGe4g0x8HYRZvBPsVhHdljUEn2NIVq4BjFbkerQUIpm/Zg +DdIx02OYI5NaAIFItO/Nis3Jz5nu2Z6qNuFoS3FJFDYoOj0dzpqPJeaAcWErtXvM ++SUWgeExX6GjfhaknBZqlxi9dnKlC54dNuYvoS++cJEPqOba+MSSQGwlfnuzCdyy +F62ARPBopY+Udf90WuioAnwMCeKpSwughQtiue+hMZL77/ZRBIls6Kl0obsXs7X9 +SQ98POyDGCBDTtWTurQ0sR8WNh8M5mQ5Fkzc4P4dyKliPUDqysU0ArSuiYgzNdws +E3PYJ/HQcu51OyLemGhmW/HGY0dVHLqlCFF1pkgl +-----END CERTIFICATE----- + +# Operating CA: Google Trust Services LLC +# Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R2 +# Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R2 +# Label: "GTS Root R2" +# Serial: 6e:47:a9:c6:5a:b3:e7:20:c5:30:9a:3f:68:52:f2:6f +# MD5 Fingerprint: 44:ED:9A:0E:A4:09:3B:00:F2:AE:4C:A3:C6:61:B0:8B +# SHA1 Fingerprint: D2:73:96:2A:2A:5E:39:9F:73:3F:E1:C7:1E:64:3F:03:38:34:FC:4D +# SHA256 Fingerprint: C4:5D:7B:B0:8E:6D:67:E6:2E:42:35:11:0B:56:4E:5F:78:FD:92:EF:05:8C:84:0A:EA:4E:64:55:D7:58:5C:60 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxlqz5yDFMJo/aFLybzANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3Lv +CvptnfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3Kg +GjSY6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9Bu +XvAuMC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOd +re7kRXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXu +PuWgf9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1 +mKPV+3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K +8YzodDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqj +x5RWIr9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsR +nTKaG73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0 +kzCqgc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9Ok +twIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBALZp +8KZ3/p7uC4Gt4cCpx/k1HUCCq+YEtN/L9x0Pg/B+E02NjO7jMyLDOfxA325BS0JT +vhaI8dI4XsRomRyYUpOM52jtG2pzegVATX9lO9ZY8c6DR2Dj/5epnGB3GFW1fgiT +z9D2PGcDFWEJ+YF59exTpJ/JjwGLc8R3dtyDovUMSRqodt6Sm2T4syzFJ9MHwAiA +pJiS4wGWAqoC7o87xdFtCjMwc3i5T1QWvwsHoaRc5svJXISPD+AVdyx+Jn7axEvb +pxZ3B7DNdehyQtaVhJ2Gg/LkkM0JR9SLA3DaWsYDQvTtN6LwG1BUSw7YhN4ZKJmB +R64JGz9I0cNv4rBgF/XuIwKl2gBbbZCr7qLpGzvpx0QnRY5rn/WkhLx3+WuXrD5R +RaIRpsyF7gpo8j5QOHokYh4XIDdtak23CZvJ/KRY9bb7nE4Yu5UC56GtmwfuNmsk +0jmGwZODUNKBRqhfYlcsu2xkiAhu7xNUX90txGdj08+JN7+dIPT7eoOboB6BAFDC +5AwiWVIQ7UNWhwD4FFKnHYuTjKJNRn8nxnGbJN7k2oaLDX5rIMHAnuFl2GqjpuiF +izoHCBy69Y9Vmhh1fuXsgWbRIXOhNUQLgD1bnF5vKheW0YMjiGZt5obicDIvUiLn +yOd/xCxgXS/Dr55FBcOEArf9LAhST4Ldo/DUhgkC +-----END CERTIFICATE----- + +# Operating CA: Google Trust Services LLC +# Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R3 +# Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R3 +# Label: "GTS Root R3" +# Serial: 6e:47:a9:c7:6c:a9:73:24:40:89:0f:03:55:dd:8d:1d +# MD5 Fingerprint: 1A:79:5B:6B:04:52:9C:5D:C7:74:33:1B:25:9A:F9:25 +# SHA1 Fingerprint: 30:D4:24:6F:07:FF:DB:91:89:8A:0B:E9:49:66:11:EB:8C:5E:46:E5 +# SHA256 Fingerprint: 15:D5:B8:77:46:19:EA:7D:54:CE:1C:A6:D0:B0:C4:03:E0:37:A9:17:F1:31:E8:A0:4E:1E:6B:7A:71:BA:BC:E5 +-----BEGIN CERTIFICATE----- +MIICDDCCAZGgAwIBAgIQbkepx2ypcyRAiQ8DVd2NHTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout +736GjOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2A +DDL24CejQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEAgFuk +fCPAlaUs3L6JbyO5o91lAFJekazInXJ0glMLfalAvWhgxeG4VDvBNhcl2MG9AjEA +njWSdIUlUfUk7GRSJFClH9voy8l27OyCbvWFGFPouOOaKaqW04MjyaR7YbPMAuhd +-----END CERTIFICATE----- + +# Operating CA: Google Trust Services LLC +# Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R4 +# Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R4 +# Label: "GTS Root R4" +# Serial: 6e:47:a9:c8:8b:94:b6:e8:bb:3b:2a:d8:a2:b2:c1:99 +# MD5 Fingerprint: 5D:B6:6A:C4:60:17:24:6A:1A:99:A8:4B:EE:5E:B4:26 +# SHA1 Fingerprint: 2A:1D:60:27:D9:4A:B1:0A:1C:4D:91:5C:CD:33:A0:CB:3E:2D:54:CB +# SHA256 Fingerprint: 71:CC:A5:39:1F:9E:79:4B:04:80:25:30:B3:63:E1:21:DA:8A:30:43:BB:26:66:2F:EA:4D:CA:7F:C9:51:A4:BD +-----BEGIN CERTIFICATE----- +MIICCjCCAZGgAwIBAgIQbkepyIuUtui7OyrYorLBmTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzu +hXyiQHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/l +xKvRHYqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNnADBkAjBqUFJ0 +CMRw3J5QdCHojXohw0+WbhXRIjVhLfoIN+4Zba3bssx9BzT1YBkstTTZbyACMANx +sbqjYAuG7ZoIapVon+Kz4ZNkfF6Tpt95LY2F45TPI11xzPKwTdb+mciUqXWi4w== +-----END CERTIFICATE----- + +# Operating CA: Sectigo +# Issuer: CN=AAA Certificate Services O=Comodo CA Limited +# Subject: CN=AAA Certificate Services O=Comodo CA Limited +# Label: "Comodo AAA Services root" +# Serial: 1 +# MD5 Fingerprint: 49:79:04:b0:eb:87:19:ac:47:b0:bc:11:51:9b:74:d0 +# SHA1 Fingerprint: d1:eb:23:a4:6d:17:d6:8f:d9:25:64:c2:f1:f1:60:17:64:d8:e3:49 +# SHA256 Fingerprint: d7:a7:a0:fb:5d:7e:27:31:d7:71:e9:48:4e:bc:de:f7:1d:5f:0c:3e:0a:29:48:78:2b:c8:3e:e0:ea:69:9e:f4 +-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj +YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM +GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua +BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe +3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 +YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR +rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm +ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU +oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF +MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v +QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t +b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF +AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q +GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz +Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 +G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi +l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 +smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== +-----END CERTIFICATE----- + +# Operating CA: Sectigo +# Issuer: CN=COMODO Certification Authority O=COMODO CA Limited +# Subject: CN=COMODO Certification Authority O=COMODO CA Limited +# Label: "COMODO Certification Authority" +# Serial: 104350513648249232941998508985834464573 +# MD5 Fingerprint: 5c:48:dc:f7:42:72:ec:56:94:6d:1c:cc:71:35:80:75 +# SHA1 Fingerprint: 66:31:bf:9e:f7:4f:9e:b6:c9:d5:a6:0c:ba:6a:be:d1:f7:bd:ef:7b +# SHA256 Fingerprint: 0c:2c:d6:3d:f7:80:6f:a3:99:ed:e8:09:11:6b:57:5b:f8:79:89:f0:65:18:f9:80:8c:86:05:03:17:8b:af:66 +-----BEGIN CERTIFICATE----- +MIIEHTCCAwWgAwIBAgIQToEtioJl4AsC7j41AkblPTANBgkqhkiG9w0BAQUFADCB +gTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxJzAlBgNV +BAMTHkNPTU9ETyBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjEyMDEwMDAw +MDBaFw0yOTEyMzEyMzU5NTlaMIGBMQswCQYDVQQGEwJHQjEbMBkGA1UECBMSR3Jl +YXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRowGAYDVQQKExFDT01P +RE8gQ0EgTGltaXRlZDEnMCUGA1UEAxMeQ09NT0RPIENlcnRpZmljYXRpb24gQXV0 +aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0ECLi3LjkRv3 +UcEbVASY06m/weaKXTuH+7uIzg3jLz8GlvCiKVCZrts7oVewdFFxze1CkU1B/qnI +2GqGd0S7WWaXUF601CxwRM/aN5VCaTwwxHGzUvAhTaHYujl8HJ6jJJ3ygxaYqhZ8 +Q5sVW7euNJH+1GImGEaaP+vB+fGQV+useg2L23IwambV4EajcNxo2f8ESIl33rXp ++2dtQem8Ob0y2WIC8bGoPW43nOIv4tOiJovGuFVDiOEjPqXSJDlqR6sA1KGzqSX+ +DT+nHbrTUcELpNqsOO9VUCQFZUaTNE8tja3G1CEZ0o7KBWFxB3NH5YoZEr0ETc5O +nKVIrLsm9wIDAQABo4GOMIGLMB0GA1UdDgQWBBQLWOWLxkwVN6RAqTCpIb5HNlpW +/zAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zBJBgNVHR8EQjBAMD6g +PKA6hjhodHRwOi8vY3JsLmNvbW9kb2NhLmNvbS9DT01PRE9DZXJ0aWZpY2F0aW9u +QXV0aG9yaXR5LmNybDANBgkqhkiG9w0BAQUFAAOCAQEAPpiem/Yb6dc5t3iuHXIY +SdOH5EOC6z/JqvWote9VfCFSZfnVDeFs9D6Mk3ORLgLETgdxb8CPOGEIqB6BCsAv +IC9Bi5HcSEW88cbeunZrM8gALTFGTO3nnc+IlP8zwFboJIYmuNg4ON8qa90SzMc/ +RxdMosIGlgnW2/4/PEZB31jiVg88O8EckzXZOFKs7sjsLjBOlDW0JB9LeGna8gI4 +zJVSk/BwJVmcIGfE7vmLV2H0knZ9P4SNVbfo5azV8fUZVqZa+5Acr5Pr5RzUZ5dd +BA6+C4OmF4O5MBKgxTMVBbkN+8cFduPYSo38NBejxiEovjBFMR7HeL5YYTisO+IB +ZQ== +-----END CERTIFICATE----- + +# Operating CA: Sectigo +# Issuer: CN=COMODO ECC Certification Authority O=COMODO CA Limited +# Subject: CN=COMODO ECC Certification Authority O=COMODO CA Limited +# Label: "COMODO ECC Certification Authority" +# Serial: 41578283867086692638256921589707938090 +# MD5 Fingerprint: 7c:62:ff:74:9d:31:53:5e:68:4a:d5:78:aa:1e:bf:23 +# SHA1 Fingerprint: 9f:74:4e:9f:2b:4d:ba:ec:0f:31:2c:50:b6:56:3b:8e:2d:93:c3:11 +# SHA256 Fingerprint: 17:93:92:7a:06:14:54:97:89:ad:ce:2f:8f:34:f7:f0:b6:6d:0f:3a:e3:a3:b8:4d:21:ec:15:db:ba:4f:ad:c7 +-----BEGIN CERTIFICATE----- +MIICiTCCAg+gAwIBAgIQH0evqmIAcFBUTAGem2OZKjAKBggqhkjOPQQDAzCBhTEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMT +IkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwMzA2MDAw +MDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdy +ZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09N +T0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlv +biBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQDR3svdcmCFYX7deSR +FtSrYpn1PlILBs5BAH+X4QokPB0BBO490o0JlwzgdeT6+3eKKvUDYEs2ixYjFq0J +cfRK9ChQtP6IHG4/bC8vCVlbpVsLM5niwz2J+Wos77LTBumjQjBAMB0GA1UdDgQW +BBR1cacZSBm8nZ3qQUfflMRId5nTeTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/ +BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjEA7wNbeqy3eApyt4jf/7VGFAkK+qDm +fQjGGoe9GKhzvSbKYAydzpmfz1wPMOG+FDHqAjAU9JM8SaczepBGR7NjfRObTrdv +GDeAU/7dIOA1mjbRxwG55tzd8/8dLDoWV9mSOdY= +-----END CERTIFICATE----- + +# Operating CA: Sectigo +# Issuer: CN=COMODO RSA Certification Authority O=COMODO CA Limited +# Subject: CN=COMODO RSA Certification Authority O=COMODO CA Limited +# Label: "COMODO RSA Certification Authority" +# Serial: 101909084537582093308941363524873193117 +# MD5 Fingerprint: 1b:31:b0:71:40:36:cc:14:36:91:ad:c4:3e:fd:ec:18 +# SHA1 Fingerprint: af:e5:d2:44:a8:d1:19:42:30:ff:47:9f:e2:f8:97:bb:cd:7a:8c:b4 +# SHA256 Fingerprint: 52:f0:e1:c4:e5:8e:c6:29:29:1b:60:31:7f:07:46:71:b8:5d:7e:a8:0d:5b:07:27:34:63:53:4b:32:b4:02:34 +-----BEGIN CERTIFICATE----- +MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB +hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV +BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5 +MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT +EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR +Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR +6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X +pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC +9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV +/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf +Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z ++pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w +qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah +SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC +u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf +Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq +crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E +FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl +wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM +4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV +2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna +FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ +CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK +boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke +jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL +S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb +QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl +0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB +NVOFBkpdn627G190 +-----END CERTIFICATE----- + +# Operating CA: Sectigo +# Issuer: CN=USERTrust ECC Certification Authority O=The USERTRUST Network +# Subject: CN=USERTrust ECC Certification Authority O=The USERTRUST Network +# Label: "USERTrust ECC Certification Authority" +# Serial: 123013823720199481456569720443997572134 +# MD5 Fingerprint: fa:68:bc:d9:b5:7f:ad:fd:c9:1d:06:83:28:cc:24:c1 +# SHA1 Fingerprint: d1:cb:ca:5d:b2:d5:2a:7f:69:3b:67:4d:e5:f0:5a:1d:0c:95:7d:f0 +# SHA256 Fingerprint: 4f:f4:60:d5:4b:9c:86:da:bf:bc:fc:57:12:e0:40:0d:2b:ed:3f:bc:4d:4f:bd:aa:86:e0:6a:dc:d2:a9:ad:7a +-----BEGIN CERTIFICATE----- +MIICjzCCAhWgAwIBAgIQXIuZxVqUxdJxVt7NiYDMJjAKBggqhkjOPQQDAzCBiDEL +MAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNl +eSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMT +JVVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMjAx +MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT +Ck5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVUaGUg +VVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBFQ0MgQ2VydGlm +aWNhdGlvbiBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQarFRaqflo +I+d61SRvU8Za2EurxtW20eZzca7dnNYMYf3boIkDuAUU7FfO7l0/4iGzzvfUinng +o4N+LZfQYcTxmdwlkWOrfzCjtHDix6EznPO/LlxTsV+zfTJ/ijTjeXmjQjBAMB0G +A1UdDgQWBBQ64QmG1M8ZwpZ2dEl23OA1xmNjmjAOBgNVHQ8BAf8EBAMCAQYwDwYD +VR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjA2Z6EWCNzklwBBHU6+4WMB +zzuqQhFkoJ2UOQIReVx7Hfpkue4WQrO/isIJxOzksU0CMQDpKmFHjFJKS04YcPbW +RNZu9YO6bVi9JNlWSOrvxKJGgYhqOkbRqZtNyWHa0V1Xahg= +-----END CERTIFICATE----- + +# Operating CA: Sectigo +# Issuer: CN=USERTrust RSA Certification Authority O=The USERTRUST Network +# Subject: CN=USERTrust RSA Certification Authority O=The USERTRUST Network +# Label: "USERTrust RSA Certification Authority" +# Serial: 2645093764781058787591871645665788717 +# MD5 Fingerprint: 1b:fe:69:d1:91:b7:19:33:a3:72:a8:0f:e1:55:e5:b5 +# SHA1 Fingerprint: 2b:8f:1b:57:33:0d:bb:a2:d0:7a:6c:51:f7:0e:e9:0d:da:b9:ad:8e +# SHA256 Fingerprint: e7:93:c9:b0:2f:d8:aa:13:e2:1c:31:22:8a:cc:b0:81:19:64:3b:74:9c:89:89:64:b1:74:6d:46:c3:d4:cb:d2 +-----BEGIN CERTIFICATE----- +MIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB +iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl +cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV +BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw +MjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU +aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy +dGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B +3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY +tJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/ +Fp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2 +VN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT +79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6 +c0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT +Yo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l +c6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee +UB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE +Hg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd +BgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G +A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF +Up/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO +VWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3 +ATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs +8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR +iQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze +Sf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ +XHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/ +qS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB +VXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB +L6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG +jjxDah2nGN59PRbxYvnKkKj9 +-----END CERTIFICATE----- diff --git a/osprey_coordinator/src/gcloud/pubsub.rs b/osprey_coordinator/src/gcloud/pubsub.rs new file mode 100644 index 0000000..57a92e6 --- /dev/null +++ b/osprey_coordinator/src/gcloud/pubsub.rs @@ -0,0 +1,587 @@ +use crate::metrics::{ + define_metrics, emit_worker::SpawnEmitWorker, string_interner::StringInterner, SharedMetrics, +}; +use crate::tokio_utils::AbortOnDrop; +use anyhow::Result; +use prost::Message; +use std::{ + collections::HashMap, + fmt::Display, + marker::PhantomData, + mem::take, + sync::{mpsc::SendError, Arc}, + time::Duration, +}; +// Use backoff 0.4 explicitly (there's also 0.1.6 in the dep tree) +extern crate backoff as backoff_v04; +use backoff_v04 as backoff; +use tokio::{ + sync::{broadcast, mpsc}, + time::MissedTickBehavior, +}; +use tonic::codegen::InterceptedService; + +use crate::gcloud::{ + auth::AuthorizationHeaderInterceptor, + google::pubsub::v1::{self as proto, publisher_client, subscriber_client}, + grpc::connection::Connection, +}; + +pub const GOOGLE_PUBSUB_DOMAIN: &str = "pubsub.googleapis.com"; + +impl Connection { + /// Creates a pubsub publisher for a specific topic. The publisher ends when all handles are dropped. + /// + /// Messages are published when: + /// - `max_interval` time has passed + /// - `max_buffer_size` for the queue buffer has been reached + pub fn create_pubsub_publisher( + &self, + topic: PubSubTopic, + max_interval: Duration, + max_buffer_size: usize, + metrics: SharedMetrics, + ) -> TopicPublisherHandle { + let publisher = TopicPublisher { + client: self.create_publisher_client(), + topic, + pubsub_stats: PubsubStats::new(), + string_interner: StringInterner::new(8), + }; + + let (send_queue_tx, closed_rx) = + publisher.spawn_send_queue_flusher(max_interval, max_buffer_size, metrics); + + TopicPublisherHandle { + send_queue_tx, + closed_rx, + } + } + + /// Creates a **synchronous** pubsub publisher for a specific topic. + /// + /// Synchronous, not meaning "blocking", but rather, meaning, that there is no batching. When you call `publish`, + /// if it succeeds, it means that your message was accepted by the remote server. + pub fn create_synchronous_pubsub_publisher( + &self, + topic: PubSubTopic, + metrics: SharedMetrics, + ) -> SynchronousTopicPublisher { + SynchronousTopicPublisher::new(topic, self.create_publisher_client(), metrics) + } + + /// Creates a subscriber client, which is used to provide a client to `pub_sub_streaming_pull::StreamingPullManager` + pub fn create_subscriber_client( + &self, + ) -> subscriber_client::SubscriberClient< + InterceptedService, + > { + subscriber_client::SubscriberClient::with_interceptor( + self.channel.clone(), + self.authorization_header_interceptor.clone(), + ) + } + + /// Creates a publisher client, which is used to provide a client to `pub_sub_streaming_pull::StreamingPullManager` + pub fn create_publisher_client( + &self, + ) -> publisher_client::PublisherClient< + InterceptedService, + > { + publisher_client::PublisherClient::with_interceptor( + self.channel.clone(), + self.authorization_header_interceptor.clone(), + ) + } +} + +/// Represents a pub-sub subscription +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct PubSubSubscription { + project: String, + subscription: String, +} + +impl PubSubSubscription { + pub fn new(project: impl Into, subscription: impl Into) -> Self { + Self { + project: project.into(), + subscription: subscription.into(), + } + } + + pub fn project(&self) -> &str { + &self.project + } + + pub fn subscription(&self) -> &str { + &self.subscription + } +} + +impl Display for PubSubSubscription { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "projects/{}/subscriptions/{}", + self.project, self.subscription + ) + } +} + +/// Represents a pub-sub topic +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct PubSubTopic { + project: String, + topic: String, +} + +impl PubSubTopic { + pub fn new(project: impl Into, topic: impl Into) -> Self { + Self { + project: project.into(), + topic: topic.into(), + } + } + + pub fn project(&self) -> &str { + &self.project + } + + pub fn topic(&self) -> &str { + &self.topic + } +} + +impl Display for PubSubTopic { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "projects/{}/topics/{}", self.project, self.topic) + } +} + +pub struct TopicPublisher { + client: publisher_client::PublisherClient< + InterceptedService, + >, + topic: PubSubTopic, + pubsub_stats: Arc, + string_interner: StringInterner, +} + +impl TopicPublisher { + // FIXME: To make this a more generalizable abstraction, we should add a few things: + // - the ability to "close" the sender, + // - ability to wait for the send queue after close to finish flushing, to allow an application to delay shutdown + // until all messages are successfully delivered to pub-sub. + + /// Starts a background task that flushes the queue when: + /// - `max_queue_flush_interval` time has passed, or: + /// - `max_buffer_size` for the queue buffer has been reached. + fn spawn_send_queue_flusher( + mut self, + max_queue_flush_interval: Duration, + max_buffer_size: usize, + metrics: SharedMetrics, + ) -> ( + mpsc::UnboundedSender, + broadcast::Receiver<()>, + ) { + let (send_queue_tx, mut send_queue_rx) = mpsc::unbounded_channel(); + + // We use a broadcast channel to notify any outstanding [`TopicPublisherHandle`] instances that we have flushed + // everything. The sender (closed_tx) is moved into the background thread and dropped when the publisher shuts + // down. + let (closed_tx, closed_rx) = broadcast::channel(1); + + tokio::spawn(async move { + let _closed_tx = closed_tx; + let mut interval = tokio::time::interval(max_queue_flush_interval); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + let _metrics_emitter = self.pubsub_stats.clone().spawn_emit_worker(metrics); + + let mut messages = Vec::new(); + + tracing::info!( + "Pubsub: publisher created for {} with {} buffer @ {:?} flush interval", + self.topic, + max_buffer_size, + max_queue_flush_interval + ); + + let mut sender_dropped = false; + + while !sender_dropped { + let should_flush_queue = tokio::select! { + // Force a flush when the interval is elapsed, if we have something in the queue. + _ = interval.tick() => !messages.is_empty(), + message = send_queue_rx.recv() => { + if let Some(message) = message { + messages.push(message); + messages.len() >= max_buffer_size + } else { + // Getting None back from send_queue_rx, means that we have no more senders, so we can stop, + // after we've completed all our work. + tracing::info!( + "Pubsub: publisher for {} shutting down, because no more senders are alive, {} messages in queue to flush, before shutdown", + self.topic, + messages.len() + ); + + sender_dropped = true; + !messages.is_empty() + } + + }, + }; + + if should_flush_queue { + // Reset since we are flushing, and this could've been triggered + // by a request rather than the interval + interval.reset(); + + let req = proto::PublishRequest { + topic: self.topic.to_string(), + messages: take(&mut messages), + }; + + self.send_publish(req).await; + } + } + }); + + (send_queue_tx, closed_rx) + } + + async fn send_publish(&mut self, req: proto::PublishRequest) { + let queue_size = req.messages.len() as i64; + + let req_ref = &req; + let publish = || { + // NOTE(rossdylan): In order to handle the lifetime requirements on future we return here, we need to either + // clone or take references to the fields we need out of self. + let mut client_clone = self.client.clone(); + let stats = self.pubsub_stats.clone(); + let project = self.string_interner.intern(&self.topic.project); + let topic = self.string_interner.intern(&self.topic.topic); + async move { + // NOTE(rossdylan): Tonic requires a brand new message for every attempt so we unfortunately need this clone + match client_clone.publish(req_ref.clone()).await { + Err(e) => { + tracing::error!("failed to flush pubsub messages due to {:?}", e); + stats.publish_failed.incr_by(queue_size, project, topic); + // Mapping of transient/permanent errors taken from + // https://cloud.google.com/pubsub/docs/reference/error-codes + let err = match e.code() { + tonic::Code::NotFound => backoff::Error::Permanent(e), + tonic::Code::AlreadyExists => backoff::Error::Permanent(e), + tonic::Code::PermissionDenied => backoff::Error::Permanent(e), + tonic::Code::FailedPrecondition => backoff::Error::Permanent(e), + tonic::Code::Unauthenticated => backoff::Error::Permanent(e), + _ => backoff::Error::transient(e), + }; + Err(err) + } + _ => { + stats.publish_sends.incr_by(queue_size, project, topic); + Ok(()) + } + } + } + }; + let final_res = + backoff::future::retry(backoff::ExponentialBackoff::default(), publish).await; + if let Err(err) = final_res { + tracing::error!("permanent failure to flush {queue_size} messages due to {err:?}"); + } + } +} + +#[derive(Debug, Clone)] +pub struct PubSubMessage { + data: Vec, + attributes: HashMap, + ordering_key: Option, + _state: PhantomData, +} + +// Two different type-states to represent a message that is fully formed (valid), versus partially formed (invalid), +// a message is fully formed when it has a non-empty data, or when it has an empty data, but non-empty attributes. +pub struct PubSubMessageInvalid; + +#[derive(Clone, Debug)] +pub struct PubSubMessageValid; + +impl PubSubMessage { + /// Constructs a message with no data, you must call [`PubSubMessage::set_attribute`] at least once, before you + /// can publish this message. If data is empty, an attribute, *must* be specified. + pub fn empty() -> Self { + Self { + data: Default::default(), + attributes: Default::default(), + ordering_key: Default::default(), + _state: PhantomData, + } + } +} + +/// Returned from `PubsubMessage::with_data` when the provided data is an empty Vec. +#[derive(Debug, Clone)] +pub struct PubsubMessageDataEmpty; + +impl std::fmt::Display for PubsubMessageDataEmpty { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("creating an empty PubsubMessage via with_data is not supported, use PubsubMessage::empty() instead") + } +} +impl std::error::Error for PubsubMessageDataEmpty {} + +impl PubSubMessage { + /// Constructs a message with the given data. + /// + /// Returns an error if the provided `data` is Empty. Prefer to use `PubsubMessage::empty()` instead. + pub fn with_data(data: impl Into>) -> Result { + let data: Vec<_> = data.into(); + if data.is_empty() { + return Err(PubsubMessageDataEmpty); + } + + Ok(Self { + data, + attributes: Default::default(), + ordering_key: Default::default(), + _state: PhantomData, + }) + } + + /// Constructs a message with the given proto data. + /// + /// The program will panic if the provided proto message `data` is empty. + pub fn with_proto_data(data: T) -> Self { + PubSubMessage::with_data(data.encode_to_vec()).expect("proto message can't be empty") + } + + /// Creates a `PubSubMessage` with no data, and the given attribute set. + pub fn with_attribute(key: impl Into, value: impl Into) -> Self { + let mut attributes = HashMap::new(); + attributes.insert(key.into(), value.into()); + + Self { + attributes, + data: Default::default(), + ordering_key: Default::default(), + _state: PhantomData, + } + } + + pub fn with_serde_data(data: T) -> Result { + let data = serde_json::to_string(&data).map_err(|_| PubsubMessageDataEmpty)?; + Self::with_data(data) + } + + /// Creates a `PubSubMessage` with no data, and the given attribute item set. + pub fn with_attribute_item( + attribute_item: MessageAttributeItemT, + ) -> Self { + Self::with_attribute(MessageAttributeItemT::KEY, attribute_item.into_value()) + } + + /// Converts this to an encoded proto message. + pub fn encode_to_vec(self) -> Vec { + self.to_proto_message().encode_to_vec() + } + + // Converts this to a proto message. + fn to_proto_message(self) -> proto::PubsubMessage { + proto::PubsubMessage { + attributes: self.attributes, + data: self.data, + ordering_key: self.ordering_key.unwrap_or_default(), + // These fields are set by the server. + message_id: Default::default(), + publish_time: None, + } + } +} + +impl PubSubMessage { + /// Sets a given attribute on the message. + pub fn set_attribute( + mut self, + key: impl Into, + value: impl Into, + ) -> PubSubMessage { + self.attributes.insert(key.into(), value.into()); + + PubSubMessage { + data: self.data, + attributes: self.attributes, + ordering_key: self.ordering_key, + _state: PhantomData, + } + } + + pub fn set_attribute_item( + self, + attribute_item: MessageAttributeItemT, + ) -> PubSubMessage { + self.set_attribute(MessageAttributeItemT::KEY, attribute_item.into_value()) + } + + /// Sets the ordering key for the message. + pub fn set_ordering_key(mut self, ordering_key: impl Into) -> Self { + self.ordering_key = Some(ordering_key.into()); + + self + } +} + +/// A message attribute item which encodes the key and value in a single struct. +pub trait MessageAttributeItem { + const KEY: &'static str; + + fn into_value(self) -> String; +} + +#[derive(Debug)] +pub struct TopicPublisherHandle { + closed_rx: broadcast::Receiver<()>, + send_queue_tx: mpsc::UnboundedSender, +} + +impl Clone for TopicPublisherHandle { + fn clone(&self) -> Self { + Self { + closed_rx: self.closed_rx.resubscribe(), + send_queue_tx: self.send_queue_tx.clone(), + } + } +} + +impl TopicPublisherHandle { + pub fn new( + send_queue_tx: mpsc::UnboundedSender, + closed_rx: broadcast::Receiver<()>, + ) -> Self { + Self { + send_queue_tx, + closed_rx, + } + } +} + +impl TopicPublisherHandle { + pub fn publish(&self, message: PubSubMessage) -> Result<(), SendError> { + let did_have_ordering_key = message.ordering_key.is_some(); + + self.send_queue_tx + .send(message.to_proto_message()) + .map_err(|e| { + SendError(PubSubMessage { + data: e.0.data, + attributes: e.0.attributes, + ordering_key: did_have_ordering_key.then_some(e.0.ordering_key), + _state: PhantomData, + }) + }) + } + + /// Consume this handle and wait for the underlying publisher to exit. + /// NOTE: If you clone and move this handle around this method will block until all handles have been dropped + pub async fn close(self) { + // Split our fields out so we can explicitly drop the send queue and wait for the closed notification + let Self { + mut closed_rx, + send_queue_tx, + } = self; + drop(send_queue_tx); + // This is fine because all we care about is that this completed + let _res = closed_rx.recv().await; + } + + pub async fn close_timeout(self, timeout: Duration) -> Result<()> { + // Split our fields out so we can explicitly drop the send queue and wait for the closed notification + let Self { + mut closed_rx, + send_queue_tx, + } = self; + drop(send_queue_tx); + let _res = tokio::time::timeout(timeout, closed_rx.recv()).await?; + Ok(()) + } +} + +struct SynchronousTopicPublisherInner { + topic: PubSubTopic, + client: publisher_client::PublisherClient< + InterceptedService, + >, + pubsub_stats: Arc, + string_interner: StringInterner, + _metrics_emitter: AbortOnDrop, +} + +#[derive(Clone)] +pub struct SynchronousTopicPublisher { + inner: Arc, +} + +impl SynchronousTopicPublisher { + pub fn new( + topic: PubSubTopic, + client: publisher_client::PublisherClient< + InterceptedService, + >, + metrics: SharedMetrics, + ) -> Self { + let pubsub_stats = PubsubStats::new(); + let _metrics_emitter = pubsub_stats.clone().spawn_emit_worker(metrics); + + Self { + inner: Arc::new(SynchronousTopicPublisherInner { + topic, + client, + pubsub_stats, + string_interner: StringInterner::new(8), + _metrics_emitter, + }), + } + } + + pub async fn publish(&self, messages: &[PubSubMessage]) -> Result<(), tonic::Status> { + let mut client = self.inner.client.clone(); + let num_messages = messages.len() as i64; + match client + .publish(proto::PublishRequest { + topic: self.inner.topic.to_string(), + messages: messages + .iter() + .cloned() + .map(|x| x.to_proto_message()) + .collect(), + }) + .await + { + Ok(_) => { + self.inner.pubsub_stats.publish_sends.incr_by( + num_messages, + self.inner.string_interner.intern(&self.inner.topic.project), + self.inner.string_interner.intern(&self.inner.topic.topic), + ); + Ok(()) + } + Err(err) => { + self.inner.pubsub_stats.publish_failed.incr_by( + num_messages, + self.inner.string_interner.intern(&self.inner.topic.project), + self.inner.string_interner.intern(&self.inner.topic.topic), + ); + Err(err) + } + } + } +} + +define_metrics!(PubsubStats, [ + publish_failed => DynamicCounter("discord_gcloud.pubsub.publish.failed", [project, topic]), + publish_sends => DynamicCounter("discord_gcloud.pubsub.publish.sends", [project, topic]), +]); diff --git a/osprey_coordinator/src/gcloud/root_ca_certificate.rs b/osprey_coordinator/src/gcloud/root_ca_certificate.rs new file mode 100644 index 0000000..5842517 --- /dev/null +++ b/osprey_coordinator/src/gcloud/root_ca_certificate.rs @@ -0,0 +1,20 @@ +use std::{fs::File, io::Read}; +use tonic::transport::Certificate; + +pub fn load() -> Result { + // Respect the standard GRPC_DEFAULT_SSL_ROOTS_FILE_PATH environment variable if present, + // otherwise use the built-in root certificate + let pem = match std::env::var("GRPC_DEFAULT_SSL_ROOTS_FILE_PATH").ok() { + Some(cert_file) => File::open(&cert_file) + .and_then(|mut file| { + let mut pem = Vec::new(); + file.read_to_end(&mut pem).map(|_| pem) + }) + .map_err(|err| format!("Failed to read {cert_file}: {err}"))?, + None => { + // PEM file from Google Trust Services (https://pki.goog/roots.pem) + include_bytes!("pki-goog-roots.pem").to_vec() + } + }; + Ok(Certificate::from_pem(pem)) +} diff --git a/osprey_coordinator/src/hashring/mod.rs b/osprey_coordinator/src/hashring/mod.rs new file mode 100644 index 0000000..5fefc19 --- /dev/null +++ b/osprey_coordinator/src/hashring/mod.rs @@ -0,0 +1,261 @@ +//! This crate provides and implementation of consistent hashing consistent with our go +//! implementation to allow members in both to work compatibly. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::sync::Arc; +use thiserror::Error; + +#[derive(Debug, Error)] +#[error("Node already in ring.")] +pub struct NodeAlreadyExists; + +#[derive(Debug, Error)] +#[error("Node with the given node name does not exist within the ring")] +pub struct NodeDoesNotExist; + +#[derive(Debug, Error)] +#[error("The ring contains no nodes.")] +pub struct RingEmpty; + +#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] +pub struct Node { + /// The name of the node. + name: Arc, + + /// The number of replicas for this node. + num_replicas: u32, +} + +impl Node { + pub fn new(name: impl Into>, num_replicas: u32) -> Node { + Node { + name: name.into(), + num_replicas, + } + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn name_arc(&self) -> Arc { + self.name.clone() + } + + pub fn num_replicas(&self) -> u32 { + self.num_replicas + } +} + +#[derive(Clone, Debug)] +struct Item { + // Number is derived from a hash of the node's key + ring_index: u64, + // The node name that this item belongs to. + node_name: Arc, +} + +/// Provides consistent hashing for keys to nodes. +#[derive(Debug, Default)] +pub struct HashRing { + nodes: RingNodes, + items: RingItems, +} + +#[derive(Default, Debug, Clone)] +pub struct RingNodes(BTreeMap, Node>); + +impl RingNodes { + pub fn with_nodes(nodes: impl IntoIterator) -> Self { + Self(nodes.into_iter().map(|n| (n.name.clone(), n)).collect()) + } + + /// Collects the nodes into a vec ordered by node name. + pub fn collect_nodes(&self) -> Vec { + self.0.values().cloned().collect() + } + + /// Adds a node to the collection, failing if a node with that name already exists. + pub fn add(&mut self, node: Node) -> Result<(), NodeAlreadyExists> { + if self.0.contains_key(&node.name) { + return Err(NodeAlreadyExists); + } + + self.0.insert(node.name.clone(), node); + Ok(()) + } + + /// Removes a node from the collection by its name, failing if a node with that + /// name does not exist. + pub fn remove(&mut self, node_name: &str) -> Result<(), NodeDoesNotExist> { + self.0.remove(node_name).ok_or(NodeDoesNotExist)?; + Ok(()) + } + + fn build_ring_items(&self) -> RingItems { + let mut items = Vec::with_capacity(self.num_replicas()); + for node in self.0.values() { + for idx in 0..node.num_replicas { + items.push(Item { + node_name: node.name.clone(), + ring_index: hash(format!("{}{}", node.name, idx)), + }); + } + } + items.sort_by_key(|node| node.ring_index); + + RingItems(items.into_boxed_slice()) + } + + /// Computes the sum of the replicas of all nodes in the collection. + fn num_replicas(&self) -> usize { + self.0.values().map(|node| node.num_replicas as usize).sum() + } + + /// Returns the number of nodes in the collection. + fn len(&self) -> usize { + self.0.len() + } +} + +#[derive(Default, Debug)] +struct RingItems(Box<[Item]>); + +impl RingItems { + /// Finds the node index for the corresponding key. + /// + /// The returned index is guaranteed to be a valid index that can be safely used to index into the contained items array. + /// + /// Returns an error if the ring is empty - as an empty ring cannot have a valid index returned. + #[inline(always)] + fn find_index(&self, key: u64) -> Result { + let len = self.len(); + if len == 0 { + return Err(RingEmpty); + } + + let index = match self.0.binary_search_by_key(&key, |item| item.ring_index) { + // We landed right on the node! What are the odds!? + // Since we are looking for the *next* node, advance the index by 1. + Ok(index) => index + 1, + Err(index) => index, + }; + + // Bounds checking on idx, check if it overflows and wrap around. + Ok(if index >= len { 0 } else { index }) + } + + /// Finds the corresponding [`Item`] for the given key. + /// + /// Returns an error if the ring is empty. + #[inline(always)] + fn find_next(&self, key: u64) -> Result<&Item, RingEmpty> { + let index = self.find_index(key)?; + // safety: index bounds are checked in `find_index`. + Ok(unsafe { self.0.get_unchecked(index) }) + } + + /// Returns the length of items within the hash ring items array. + #[inline(always)] + fn len(&self) -> usize { + self.0.len() + } + + /// Computes the next index by adding 1, or wrapping around if the index overflows. + #[inline(always)] + fn next_index(&self, mut index: usize) -> usize { + index += 1; + if index >= self.len() { + 0 + } else { + index + } + } +} + +impl HashRing { + /// Creates an empty hash ring with no nodes. + pub fn new() -> Self { + Default::default() + } + + /// Constructs a HashRing given an iterator of nodes. If a node is duplicated within the iterator, the + /// last value will be used. + pub fn with_nodes(nodes: impl IntoIterator) -> Self { + let nodes = RingNodes::with_nodes(nodes); + let items = nodes.build_ring_items(); + Self { items, nodes } + } + + /// Returns an immutable reference to the current nodes in the hash ring. + pub fn nodes(&self) -> &RingNodes { + &self.nodes + } + + /// Replaces the nodes in the hash ring. + pub fn set_nodes(&mut self, nodes: impl IntoIterator) { + *self = Self::with_nodes(nodes); + } + + /// Adds a node to the hash ring. + pub fn add(&mut self, node: Node) -> Result<(), NodeAlreadyExists> { + self.nodes.add(node)?; + self.items = self.nodes.build_ring_items(); + Ok(()) + } + + /// Removes a node from the hash ring. + pub fn remove(&mut self, node_name: impl AsRef) -> Result<(), NodeDoesNotExist> { + self.nodes.remove(node_name.as_ref())?; + self.items = self.nodes.build_ring_items(); + Ok(()) + } + + /// Finds a node for a given key. + #[inline(always)] + pub fn find(&self, key: impl AsRef) -> Result<&Arc, RingEmpty> { + self.items.find_next(hash(key)).map(|item| &item.node_name) + } + + /// Finds `num` number of nodes for a given key. + pub fn find_n(&self, key: impl AsRef, num: usize) -> Result>, RingEmpty> { + let num = self.nodes.len().min(num); + let mut index = self.items.find_index(hash(key.as_ref()))?; + + let mut nodes = Vec::with_capacity(num); + while nodes.len() < num { + // safety: bounds checking is done in `find_index` and `next_index`. + let item = unsafe { self.items.0.get_unchecked(index) }; + index = self.items.next_index(index); + + // Skip Nodes we have already seen. + if !nodes.contains(&item.node_name) { + nodes.push(item.node_name.clone()); + } + } + + Ok(nodes) + } +} + +fn hash(key: impl AsRef) -> u64 { + let digest = md5::compute(key.as_ref()); + + // This may seem sub-optimal, but actually, this is more fast than say doing + // u64::from_le_bytes((&digest[8..].try_into().unwrap())); + // or u64::from_ne_bytes([digest[8], digest[9], ..]) + let low = u32::from(digest[11]) << 24 + | u32::from(digest[10]) << 16 + | u32::from(digest[9]) << 8 + | u32::from(digest[8]); + let high = u32::from(digest[15]) << 24 + | u32::from(digest[14]) << 16 + | u32::from(digest[13]) << 8 + | u32::from(digest[12]); + let mut key_int = u64::from(high); + key_int <<= 32; + key_int &= 0xffff_ffff_0000_0000; + key_int |= u64::from(low); + key_int +} diff --git a/osprey_coordinator/src/label_service_client.rs b/osprey_coordinator/src/label_service_client.rs new file mode 100644 index 0000000..1fadec6 --- /dev/null +++ b/osprey_coordinator/src/label_service_client.rs @@ -0,0 +1,75 @@ +use crate::pigeon::grpc_client::GrpcClient; +use crate::pigeon::Connector; + +use crate::round_robin; +use anyhow::Result; +use tonic::transport::{Channel, Endpoint, Error as TonicError}; + +use crate::proto::label_service_client::LabelServiceClient as LabelServiceClientProto; +use crate::proto::{Entity, EntityKey, GetEntityRequest, GetEntityResponse}; + +use futures::future::try_join_all; +use futures::future::{BoxFuture, FutureExt}; + +pub struct LabelServiceClient { + rpc_client: GrpcClient>, +} + +impl LabelServiceClient { + pub async fn new() -> Result { + let rpc_client = GrpcClient::>::new( + "label_service_client".into(), + "discord_smite_labels".into(), + 2, + Some("smite_coordinator"), + ) + .await?; + Ok(LabelServiceClient { rpc_client }) + } + + pub async fn get_entities(&self, entities: Vec) -> Result> { + let requests: Vec<_> = entities + .into_iter() + .map(|entity_key| GetEntityRequest { + // the routing key is only important client-side ^^ + routing_key: "meow".to_string(), + key: Some(entity_key), + }) + .collect(); + + let mut futures: Vec>> = + Vec::with_capacity(8); + + for request in requests { + let future: BoxFuture> = + round_robin!(self.rpc_client, get_entity, request).boxed(); + futures.push(future); + } + + let get_entity_results: Vec = try_join_all(futures).await?; + + Ok(get_entity_results + .into_iter() + .filter_map(|get_entity_response| get_entity_response.entity) + .collect()) + } +} + +#[tonic::async_trait] +impl Connector for LabelServiceClientProto { + async fn connect(dst: Endpoint) -> Result { + let svc = Self::connect(dst) + .await? + .max_decoding_message_size(20 * 1024 * 1024) + .max_encoding_message_size(20 * 1024 * 1024); + + Ok(svc) + } + fn connect_lazy(dst: Endpoint) -> Self { + let channel = dst.connect_lazy(); + + Self::new(channel) + .max_decoding_message_size(20 * 1024 * 1024) + .max_encoding_message_size(20 * 1024 * 1024) + } +} diff --git a/osprey_coordinator/src/main.rs b/osprey_coordinator/src/main.rs new file mode 100644 index 0000000..527512d --- /dev/null +++ b/osprey_coordinator/src/main.rs @@ -0,0 +1,145 @@ +mod backoff_utils; +mod cached_futures; +mod coordinator_metrics; +mod discovery; +mod etcd; +mod etcd_config; +mod etcd_watcherd; +mod future_utils; +mod gcloud; +mod hashring; +mod label_service_client; +mod metrics; +mod pigeon; +mod priority_queue; +mod proto; +mod pub_sub_streaming_pull; +mod pubsub; +mod shutdown_handler; +mod signals; +mod smite_bidirectional_stream; +mod snowflake_client; +mod sync_action_rpc; +mod tokio_utils; +#[cfg(test)] +mod tonic_mock; +use anyhow::Result; +use clap::Parser; +use proto::smite_coordinator_sync_action::smite_coordinator_sync_action_service_server::SmiteCoordinatorSyncActionServiceServer; +use std::sync::Arc; +use std::time::Duration; + +use crate::snowflake_client::SnowflakeClient; +use crate::{ + coordinator_metrics::SmiteCoordinatorMetrics, label_service_client::LabelServiceClient, +}; + +use crate::metrics::emit_worker::SpawnEmitWorker; +use crate::metrics::new_client; + +use priority_queue::{create_ackable_action_priority_queue, spawn_priority_queue_metrics_worker}; +use pubsub::start_pubsub_subscriber; +use tokio::join; + +use crate::proto::smite_coordinator_service_server::SmiteCoordinatorServiceServer; +use crate::smite_bidirectional_stream::SmiteCoordinatorServer; + +#[derive(Debug, Parser)] +struct CliOptions { + #[arg( + short, + long, + default_value = "19950", + env = "SMITE_COORDINATOR_BIDI_STREAM_PORT" + )] + bidi_stream_port: u16, + #[arg( + long, + default_value = "19951", + env = "SMITE_COORDINATOR_SYNC_ACTION_PORT" + )] + sync_action_port: u16, + #[arg( + long, + default_value = "http://localhost:19952", + env = "SNOWFLAKE_API_ENDPOINT" + )] + snowflake_api_endpoint: String, +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt::init(); + let opts = CliOptions::parse(); + + tracing::info!("starting Osprey Coordinator"); + + tracing::info!("creating osprey-snowflake client"); + let snowflake_client = Arc::new(SnowflakeClient::new(opts.snowflake_api_endpoint)); + + let (priority_queue_sender, priority_queue_receiver) = create_ackable_action_priority_queue(); + let metrics = SmiteCoordinatorMetrics::new(); + tracing::info!("starting grpc metrics worker"); + let _worker_guard = metrics + .clone() + .spawn_emit_worker(new_client("smite_coordinator").unwrap()); + + let smite_coordinator_grpc_bidi_stream_service = + SmiteCoordinatorServiceServer::new(SmiteCoordinatorServer::new( + priority_queue_sender.clone(), + priority_queue_receiver.clone(), + metrics.clone(), + )); + + tracing::info!("starting label service client"); + let label_service_client = LabelServiceClient::new().await?; + + let smite_coordinator_sync_action_service = + SmiteCoordinatorSyncActionServiceServer::new(sync_action_rpc::SyncActionServer::new( + snowflake_client.clone(), + priority_queue_sender.clone(), + metrics.clone(), + label_service_client, + )); + + let pubsub_fut = start_pubsub_subscriber( + snowflake_client, + priority_queue_sender.clone(), + metrics.clone(), + ); + let grpc_bidi_stream_service_fut = pigeon::serve( + smite_coordinator_grpc_bidi_stream_service, + "smite_coordinator", + opts.bidi_stream_port, + Duration::from_secs(30), + ); + let sync_action_service_fut = pigeon::serve( + smite_coordinator_sync_action_service, + "smite_coordinator_sync_action", + opts.sync_action_port, + Duration::from_secs(60), + ); + + tracing::info!("starting priority queue metrics worker"); + let _drop_guard = + spawn_priority_queue_metrics_worker(priority_queue_sender.clone(), metrics.clone()); + + shutdown_handler::spawn_shutdown_handler( + priority_queue_sender.clone(), + priority_queue_receiver.clone(), + ); + + tracing::info!("starting pubsub listener/bidi stream/sync classification rpc"); + let (pubsub_result, grpc_bidi_stream_service_result, sync_action_service_result) = join!( + pubsub_fut, + grpc_bidi_stream_service_fut, + sync_action_service_fut + ); + tracing::info!({ + pubsub_result=?pubsub_result, + bidi_stream_result=?grpc_bidi_stream_service_result, + sync_action_result=?sync_action_service_result}, + "osprey coordinator terminated"); + + Ok(()) +} diff --git a/osprey_coordinator/src/metrics/base_metric/mod.rs b/osprey_coordinator/src/metrics/base_metric/mod.rs new file mode 100644 index 0000000..e7226e0 --- /dev/null +++ b/osprey_coordinator/src/metrics/base_metric/mod.rs @@ -0,0 +1,90 @@ +use std::fmt::Debug; +use std::hash::Hash; + +use crate::metrics::SharedMetrics; + +#[derive(Debug)] +pub struct StaticTag { + key: &'static str, + value: &'static str, +} + +impl StaticTag { + pub const fn new(key: &'static str, value: &'static str) -> Self { + Self { key, value } + } +} + +pub trait BaseMetric: Send + Sync + Default + 'static { + const NAME: &'static str; + const STATIC_TAGS: &'static [StaticTag]; + + /// Calls a reducer function `f` over the static tag pairs for this counter. + fn fold_static_tag_pairs(mut t: T, f: F) -> T + where + F: Fn(T, &'static str, &'static str) -> T, + { + for StaticTag { key, value } in Self::STATIC_TAGS { + t = f(t, key, value); + } + t + } +} + +#[async_trait::async_trait] +pub trait EmitMetrics: Send + Sync + 'static { + async fn emit_metrics(&self, metrics: &SharedMetrics); +} + +pub trait DynamicTagKey: Hash + Eq + Debug + PartialEq + Eq + Ord + PartialOrd + 'static { + fn fold_tag_pairs(&self, t: T, f: F) -> T + where + F: Fn(T, &'static str, &'static str) -> T; + + fn get_tag_keys() -> &'static [&'static str]; +} + +pub trait DynamicTagValue: 'static { + fn as_static_str(&self) -> &'static str; +} + +impl DynamicTagValue for &'static str { + #[inline(always)] + fn as_static_str(&self) -> &'static str { + self + } +} + +impl DynamicTagValue for bool { + #[inline(always)] + fn as_static_str(&self) -> &'static str { + match self { + true => "true", + false => "false", + } + } +} + +impl DynamicTagValue for tonic::Code { + fn as_static_str(&self) -> &'static str { + match self { + tonic::Code::Ok => "OK", + tonic::Code::Cancelled => "CANCELLED", + tonic::Code::Unknown => "UNKNOWN", + tonic::Code::InvalidArgument => "INVALID_ARGUMENT", + tonic::Code::DeadlineExceeded => "DEADLINE_EXCEEDED", + tonic::Code::NotFound => "NOT_FOUND", + tonic::Code::AlreadyExists => "ALREADY_EXISTS", + tonic::Code::PermissionDenied => "PERMISSION_DENIED", + tonic::Code::ResourceExhausted => "RESOURCE_EXHAUSTED", + tonic::Code::FailedPrecondition => "FAILED_PRECONDITION", + tonic::Code::Aborted => "ABORTED", + tonic::Code::OutOfRange => "OUT_OF_RANGE", + tonic::Code::Unimplemented => "UNIMPLEMENTED", + tonic::Code::Internal => "INTERNAL", + tonic::Code::Unavailable => "UNAVAILABLE", + tonic::Code::DataLoss => "DATA_LOSS", + tonic::Code::Unauthenticated => "UNAUTHENTICATED", + } + } +} diff --git a/osprey_coordinator/src/metrics/counters/dynamic_counter.rs b/osprey_coordinator/src/metrics/counters/dynamic_counter.rs new file mode 100644 index 0000000..2a57174 --- /dev/null +++ b/osprey_coordinator/src/metrics/counters/dynamic_counter.rs @@ -0,0 +1,103 @@ +use std::{ + collections::hash_map::Entry, + fmt::Debug, + sync::atomic::{AtomicI64, Ordering}, +}; + +use async_trait::async_trait; +use fxhash::FxHashMap; +use parking_lot::RwLock; + +use crate::metrics::{BaseMetric, DynamicTagKey, EmitMetrics, SharedMetrics}; + +pub type DynamicCounterStorage = RwLock>; + +/// A dynamic counter holds counters for a key/value pair for a singular metric. +pub trait DynamicCounter: BaseMetric { + type Key: DynamicTagKey; + + #[doc(hidden)] + fn storage(&self) -> &DynamicCounterStorage; + + #[doc(hidden)] + #[inline(always)] + fn decr_with_key(&self, key: Self::Key, value: i64) { + self.incr_with_key(key, -value); + } + + #[doc(hidden)] + #[inline(always)] + fn incr_with_key(&self, key: Self::Key, value: i64) { + if value == 0 { + return; + } + + let storage = self.storage(); + + // First, try the *fast* path, for a dynamic key pair we've seen before. + let locked = storage.read(); + if let Some(counter) = locked.get(&key) { + counter.fetch_add(value, Ordering::Relaxed); + return; + } + drop(locked); + + // Second, try the *slow* path, which is that we have not discovered this dynamic key pair before, + // so we must insert it into the storage. + let mut locked = storage.write(); + match locked.entry(key) { + Entry::Occupied(ent) => { + ent.get().fetch_add(value, Ordering::Relaxed); + } + Entry::Vacant(ent) => { + ent.insert(AtomicI64::new(value)); + } + } + } +} + +/// A wrapper that disambiguates between static and dynamic counters for the purpose +/// of the EmitMetrics auto trait impl. +#[derive(Default)] +pub struct DynamicCounterWrapper(Counter); + +impl std::ops::Deref for DynamicCounterWrapper { + type Target = Counter; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Debug for DynamicCounterWrapper { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DynamicCounter") + .field("name", &Counter::NAME) + .field("static_tags", &Counter::STATIC_TAGS) + .field("dynamic_tag_keys", &Counter::Key::get_tag_keys()) + .field("num_distinct_keys", &self.storage().read().len()) + .finish() + } +} + +#[async_trait] +impl EmitMetrics for DynamicCounterWrapper +where + Counter: DynamicCounter, +{ + async fn emit_metrics(&self, metrics: &SharedMetrics) { + let storage = self.storage(); + let locked = storage.read(); + for (key, value) in locked.iter() { + let count = value.swap(0, Ordering::Relaxed); + if count == 0 { + continue; + } + + let mut builder = metrics.count_with_tags(Counter::NAME, count); + builder = Counter::fold_static_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder = key.fold_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder.try_send().ok(); + } + } +} diff --git a/osprey_coordinator/src/metrics/counters/mod.rs b/osprey_coordinator/src/metrics/counters/mod.rs new file mode 100644 index 0000000..8a09992 --- /dev/null +++ b/osprey_coordinator/src/metrics/counters/mod.rs @@ -0,0 +1,6 @@ +pub mod dynamic_counter; +pub mod static_counter; + +pub use super::{DynamicTagKey, DynamicTagValue}; +pub use dynamic_counter::{DynamicCounter, DynamicCounterStorage}; +pub use static_counter::StaticCounter; diff --git a/osprey_coordinator/src/metrics/counters/static_counter.rs b/osprey_coordinator/src/metrics/counters/static_counter.rs new file mode 100644 index 0000000..c41b1b5 --- /dev/null +++ b/osprey_coordinator/src/metrics/counters/static_counter.rs @@ -0,0 +1,73 @@ +use std::{ + fmt::Debug, + sync::atomic::{AtomicI64, Ordering}, +}; + +use async_trait::async_trait; + +use crate::metrics::{BaseMetric, EmitMetrics, SharedMetrics}; + +pub trait StaticCounter: BaseMetric { + #[doc(hidden)] + fn value(&self) -> &AtomicI64; + + #[inline(always)] + fn incr_by(&self, value: i64) { + self.value().fetch_add(value, Ordering::Relaxed); + } + + #[inline(always)] + fn decr_by(&self, value: i64) { + self.incr_by(-value); + } + + #[inline(always)] + fn incr(&self) { + self.incr_by(1); + } + + #[inline(always)] + fn decr(&self) { + self.decr_by(1); + } +} + +/// A wrapper that disambiguates between static and dynamic counters for the purpose +/// of the EmitMetrics auto trait impl. +#[derive(Default)] +pub struct StaticCounterWrapper(Counter); + +impl std::ops::Deref for StaticCounterWrapper { + type Target = Counter; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Debug for StaticCounterWrapper { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StaticCounter") + .field("name", &Counter::NAME) + .field("static_tags", &Counter::STATIC_TAGS) + .field("value", &self.value().load(Ordering::Relaxed)) + .finish() + } +} + +#[async_trait] +impl EmitMetrics for StaticCounterWrapper +where + Counter: StaticCounter, +{ + async fn emit_metrics(&self, metrics: &SharedMetrics) { + let count = self.value().swap(0, Ordering::Relaxed); + if count == 0 { + return; + } + + let mut builder = metrics.count_with_tags(Counter::NAME, count); + builder = Counter::fold_static_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder.try_send().ok(); + } +} diff --git a/osprey_coordinator/src/metrics/emit_worker/mod.rs b/osprey_coordinator/src/metrics/emit_worker/mod.rs new file mode 100644 index 0000000..426ea10 --- /dev/null +++ b/osprey_coordinator/src/metrics/emit_worker/mod.rs @@ -0,0 +1,73 @@ +use crate::tokio_utils::AbortOnDrop; +use std::{sync::Arc, time::Duration}; + +use tokio::time::{interval, MissedTickBehavior}; + +use crate::metrics::{EmitMetrics, SharedMetrics}; + +const DEFAULT_METRIC_REPORTING_FREQUENCY: Duration = Duration::from_millis(300); + +pub trait SpawnEmitWorker { + #[must_use = "spawn_emit_worker return value must be used."] + fn spawn_emit_worker(self, cadence_metrics: SharedMetrics) -> AbortOnDrop<()>; + + #[must_use = "spawn_emit_worker_with_frequency return value must be used."] + fn spawn_emit_worker_with_frequency( + self, + cadence_metrics: SharedMetrics, + frequency: Duration, + ) -> AbortOnDrop<()>; +} + +impl SpawnEmitWorker for Arc +where + T: EmitMetrics, +{ + fn spawn_emit_worker(self, cadence_metrics: SharedMetrics) -> AbortOnDrop<()> { + self.spawn_emit_worker_with_frequency(cadence_metrics, DEFAULT_METRIC_REPORTING_FREQUENCY) + } + + fn spawn_emit_worker_with_frequency( + self, + cadence_metrics: SharedMetrics, + frequency: Duration, + ) -> AbortOnDrop<()> { + let mut interval = interval(frequency); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + let join_handle = tokio::task::spawn(async move { + loop { + interval.tick().await; + self.emit_metrics(&cadence_metrics).await; + } + }); + + AbortOnDrop::new(join_handle) + } +} + +impl SpawnEmitWorker for T +where + T: EmitMetrics, +{ + fn spawn_emit_worker(self, cadence_metrics: SharedMetrics) -> AbortOnDrop<()> { + self.spawn_emit_worker_with_frequency(cadence_metrics, DEFAULT_METRIC_REPORTING_FREQUENCY) + } + + fn spawn_emit_worker_with_frequency( + self, + cadence_metrics: SharedMetrics, + frequency: Duration, + ) -> AbortOnDrop<()> { + let mut interval = interval(frequency); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + let join_handle = tokio::task::spawn(async move { + loop { + interval.tick().await; + self.emit_metrics(&cadence_metrics).await; + } + }); + AbortOnDrop::new(join_handle) + } +} diff --git a/osprey_coordinator/src/metrics/gauges/dynamic_gauge.rs b/osprey_coordinator/src/metrics/gauges/dynamic_gauge.rs new file mode 100644 index 0000000..58eb441 --- /dev/null +++ b/osprey_coordinator/src/metrics/gauges/dynamic_gauge.rs @@ -0,0 +1,132 @@ +use std::{collections::hash_map::Entry, fmt::Debug, sync::Arc}; + +use async_trait::async_trait; +use fxhash::FxHashMap; +use parking_lot::RwLock; + +use super::gauge_struct::{GaugeHandleOwned, GaugeInner, Many, One}; +use crate::metrics::base_metric::{BaseMetric, DynamicTagKey, EmitMetrics}; +use crate::metrics::SharedMetrics; + +pub type DynamicGaugeStorage = RwLock>>; + +/// A dynamic gauge holds gauges for a key/value pair for a singular metric. +pub trait DynamicGauge: BaseMetric { + type Key: DynamicTagKey; + + #[doc(hidden)] + fn storage(&self) -> &DynamicGaugeStorage; + + #[doc(hidden)] + #[inline(always)] + unsafe fn decr_with_key(&self, key: Self::Key, value: u64) { + if value == 0 { + return; + } + + let gauge_inner = get_gauge_inner_for_key(self, key); + gauge_inner.decr_by(value); + } + + #[doc(hidden)] + #[inline(always)] + unsafe fn incr_with_key(&self, key: Self::Key, value: u64) { + if value == 0 { + return; + } + + let gauge_inner = get_gauge_inner_for_key(self, key); + gauge_inner.incr_by(value); + } + + #[doc(hidden)] + #[inline(always)] + fn acquire_many_with_key_owned(&self, key: Self::Key, value: u64) -> GaugeHandleOwned { + let gauge_inner = get_gauge_inner_for_key(self, key); + gauge_inner.incr_by(value); + GaugeHandleOwned { + value: Many(value), + inner: Arc::clone(&gauge_inner), + } + } + + #[doc(hidden)] + #[inline(always)] + fn acquire_with_key_owned(&self, key: Self::Key) -> GaugeHandleOwned { + let gauge_inner = get_gauge_inner_for_key(self, key); + gauge_inner.incr_by(1); + GaugeHandleOwned { + value: One, + inner: Arc::clone(&gauge_inner), + } + } + + #[doc(hidden)] + #[inline(always)] + fn set_with_key(&self, key: Self::Key, value: u64) { + let gauge_inner = get_gauge_inner_for_key(self, key); + gauge_inner.set(value); + } +} + +#[inline(always)] +fn get_gauge_inner_for_key( + dynamic_gauge: &T, + key: ::Key, +) -> Arc { + let storage = dynamic_gauge.storage(); + + // First, try the *fast* path, for a dynamic key pair we've seen before. + let locked = storage.read(); + if let Some(gauge) = locked.get(&key) { + return Arc::clone(gauge); + } + drop(locked); + + // Second, try the *slow* path, which is that we have not discovered this dynamic key pair before, + // so we must insert it into the storage. + let mut locked = storage.write(); + match locked.entry(key) { + Entry::Occupied(ent) => Arc::clone(ent.get()), + Entry::Vacant(ent) => Arc::clone(ent.insert(Arc::new(GaugeInner::default()))), + } +} +/// A wrapper that disambiguates between static and dynamic gauges for the purpose +/// of the EmitMetrics auto trait impl. +#[derive(Default)] +pub struct DynamicGaugeWrapper(Gauge); + +impl std::ops::Deref for DynamicGaugeWrapper { + type Target = Gauge; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Debug for DynamicGaugeWrapper { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DynamicGauge") + .field("name", &Gauge::NAME) + .field("static_tags", &Gauge::STATIC_TAGS) + .field("dynamic_tag_keys", &Gauge::Key::get_tag_keys()) + .field("num_distinct_keys", &self.storage().read().len()) + .finish() + } +} + +#[async_trait] +impl EmitMetrics for DynamicGaugeWrapper +where + Gauge: DynamicGauge, +{ + async fn emit_metrics(&self, metrics: &SharedMetrics) { + let storage = self.storage(); + let locked = storage.read(); + for (key, value) in locked.iter() { + value.snapshot().emit::(metrics, |builder| { + key.fold_tag_pairs(builder, |b, k, v| b.with_tag(k, v)) + }); + } + } +} diff --git a/osprey_coordinator/src/metrics/gauges/gauge_struct.rs b/osprey_coordinator/src/metrics/gauges/gauge_struct.rs new file mode 100644 index 0000000..557bc93 --- /dev/null +++ b/osprey_coordinator/src/metrics/gauges/gauge_struct.rs @@ -0,0 +1,180 @@ +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; + +use cadence::{Gauge, MetricBuilder}; + +use crate::metrics::base_metric::BaseMetric; +use crate::metrics::SharedMetrics; + +#[derive(Debug)] +pub struct GaugeInner { + pub(crate) current: AtomicU64, + pub(crate) max: AtomicU64, + pub(crate) min: AtomicU64, +} + +impl Default for GaugeInner { + fn default() -> Self { + Self { + current: AtomicU64::new(0), + // min -> MAX, and max -> MIN are correct here. we are using the smallest/largest value in the domain + // to track: "the gauge has not changed within this interval." + max: AtomicU64::new(u64::MIN), + min: AtomicU64::new(u64::MAX), + } + } +} + +impl GaugeInner { + // Get the current value of a gauge + pub fn current(&self) -> u64 { + self.current.load(Ordering::Relaxed) + } + + pub(crate) fn min(&self, current: u64) -> u64 { + let min = self.min.load(Ordering::Relaxed); + if min == u64::MAX { + current + } else { + min + } + } + + pub(crate) fn max(&self, current: u64) -> u64 { + let max = self.max.load(Ordering::Relaxed); + if max == u64::MIN { + current + } else { + max + } + } + + pub(crate) fn swap_min(&self, current: u64) -> u64 { + let min = self.min.swap(u64::MAX, Ordering::Relaxed); + if min == u64::MAX { + current + } else { + min + } + } + + pub(crate) fn swap_max(&self, current: u64) -> u64 { + let max = self.max.swap(u64::MIN, Ordering::Relaxed); + if max == u64::MIN { + current + } else { + max + } + } + + pub(crate) fn decr_by(&self, value: u64) { + self.current.fetch_sub(value, Ordering::Relaxed); + + // Track min/max values in the reporting window. + let current = self.current(); + self.min.fetch_min(current, Ordering::Relaxed); + self.max.fetch_max(current, Ordering::Relaxed); + } + + pub(crate) fn incr_by(&self, value: u64) { + self.current.fetch_add(value, Ordering::Relaxed); + + // Track min/max values in the reporting window. + let current = self.current(); + self.min.fetch_min(current, Ordering::Relaxed); + self.max.fetch_max(current, Ordering::Relaxed); + } + + pub(crate) fn set(&self, value: u64) { + self.current.store(value, Ordering::Relaxed); + + // Track min/max values in the reporting window. + self.min.fetch_min(value, Ordering::Relaxed); + self.max.fetch_max(value, Ordering::Relaxed); + } + + pub(crate) fn snapshot(&self) -> GaugeSnapshot { + let current = self.current(); + let mut min = self.swap_min(current); + let mut max = self.swap_max(current); + + // In the rare case that these can end up swapped due to de-sync between resetting the min and max, and an active + // metrics recorder, just swap the min/max so they are sensible for this metrics window. + if min > max { + std::mem::swap(&mut min, &mut max); + } + + GaugeSnapshot { current, min, max } + } +} + +pub(crate) struct GaugeSnapshot { + pub(crate) current: u64, + pub(crate) min: u64, + pub(crate) max: u64, +} + +impl GaugeSnapshot { + pub(crate) fn emit( + self, + metrics: &SharedMetrics, + apply_dynamic_tags: TApplyDynamicTags, + ) where + M: BaseMetric, + TApplyDynamicTags: for<'a> Fn(MetricBuilder<'a, 'a, Gauge>) -> MetricBuilder<'a, 'a, Gauge>, + { + { + let mut builder = metrics.gauge_with_tags(::NAME, self.current); + builder = ::fold_static_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder = apply_dynamic_tags(builder); + builder.try_send().ok(); + } + { + let metric_name = format!("{}.min", ::NAME); + let mut builder = metrics.gauge_with_tags(&metric_name, self.min); + builder = ::fold_static_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder = apply_dynamic_tags(builder); + builder.try_send().ok(); + } + { + let metric_name = format!("{}.max", ::NAME); + let mut builder = metrics.gauge_with_tags(&metric_name, self.max); + builder = ::fold_static_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder = apply_dynamic_tags(builder); + builder.try_send().ok(); + } + } +} +pub struct One; +pub struct Many(pub u64); + +pub trait StoredValue: Send { + fn get(&self) -> u64; +} + +impl StoredValue for One { + fn get(&self) -> u64 { + 1 + } +} + +impl StoredValue for Many { + fn get(&self) -> u64 { + self.0 + } +} + +pub struct GaugeHandleOwned { + pub(crate) value: Value, + pub(crate) inner: Arc, +} + +impl Drop for GaugeHandleOwned { + fn drop(&mut self) { + self.inner + .current + .fetch_sub(self.value.get(), Ordering::Relaxed); + } +} diff --git a/osprey_coordinator/src/metrics/gauges/mod.rs b/osprey_coordinator/src/metrics/gauges/mod.rs new file mode 100644 index 0000000..ef9511f --- /dev/null +++ b/osprey_coordinator/src/metrics/gauges/mod.rs @@ -0,0 +1,8 @@ +pub mod dynamic_gauge; +mod gauge_struct; +pub mod static_gauge; + +pub use super::{DynamicTagKey, DynamicTagValue}; +pub use dynamic_gauge::{DynamicGauge, DynamicGaugeStorage}; +pub use gauge_struct::{GaugeHandleOwned, GaugeInner, Many, One}; +pub use static_gauge::StaticGauge; diff --git a/osprey_coordinator/src/metrics/gauges/static_gauge.rs b/osprey_coordinator/src/metrics/gauges/static_gauge.rs new file mode 100644 index 0000000..e03514a --- /dev/null +++ b/osprey_coordinator/src/metrics/gauges/static_gauge.rs @@ -0,0 +1,128 @@ +use std::{fmt::Debug, sync::Arc}; + +use async_trait::async_trait; + +use super::{GaugeHandleOwned, GaugeInner, Many, One}; +use crate::metrics::base_metric::{BaseMetric, EmitMetrics}; +use crate::metrics::SharedMetrics; + +pub trait StaticGauge: BaseMetric { + #[doc(hidden)] + fn inner(&self) -> &GaugeInner; + + #[doc(hidden)] + fn inner_arc(&self) -> Arc; + + /// Increments the gauge by a given value. + /// + /// # Safety + /// This method must be called with an accompanied [`StaticGauge::decr_by`], otherwise the state of the gauge + /// will be incorrect. + /// + /// More conveniently and safely though, the use of [`StaticGauge::acquire_owned`] + /// or [`StaticGauge::acquire_many_owned`] is recommended, + /// which automatically decrements the gauge for you when the returned [`GaugeHandle`] falls out of scope. + #[inline(always)] + unsafe fn incr_by(&self, value: u64) { + self.inner().incr_by(value); + } + + /// Decrements the gauge by a given value. + /// + /// # Safety + /// The gauge value after decrement could underflow. + /// + /// See [`StaticGauge::incr_by`] for more information. + #[inline(always)] + unsafe fn decr_by(&self, value: u64) { + self.inner().decr_by(value); + } + + /// Increments the gauge by 1. + /// + /// # Safety + /// This method must be called with an accompanied [`StaticGauge::decr`], otherwise the state of the gauge + /// will be incorrect. + /// + /// See [`StaticGauge::incr_by`] for more information. + #[inline(always)] + unsafe fn incr(&self) { + self.incr_by(1); + } + + /// Decrements the gauge by 1. + /// + /// # Safety + /// The gauge value after decrement could underflow. + /// + /// See [`StaticGauge::incr_by`] for more information. + #[inline(always)] + unsafe fn decr(&self) { + self.decr_by(1); + } + + /// Increments the gauge by a given value and returns a [`GaugeHandleOwned`] which will automatically decrement + /// the gauge by that value when dropped. + #[inline(always)] + fn acquire_many_owned(&self, value: u64) -> GaugeHandleOwned { + unsafe { self.incr_by(value) }; + + GaugeHandleOwned { + value: Many(value), + inner: self.inner_arc(), + } + } + + /// Increments the gauge by 1 and returns a [`GaugeHandleOwned`] which will automatically decrement + /// the gauge when dropped. + #[inline(always)] + fn acquire_owned(&self) -> GaugeHandleOwned { + unsafe { self.incr() }; + + GaugeHandleOwned { + value: One, + inner: self.inner_arc(), + } + } + + /// Sets the gauge to a given value. This method is useful when you have a background task that periodically + /// reports a given value. + #[inline(always)] + fn set(&self, value: u64) { + self.inner().set(value); + } +} + +#[derive(Default)] +pub struct StaticGaugeWrapper(Gauge); + +impl std::ops::Deref for StaticGaugeWrapper { + type Target = Gauge; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Debug for StaticGaugeWrapper { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let value = self.inner(); + let current = value.current(); + f.debug_struct("StaticGauge") + .field("name", &Gauge::NAME) + .field("static_tags", &Gauge::STATIC_TAGS) + .field("current", ¤t) + .field("min", &value.min(current)) + .field("max", &value.max(current)) + .finish() + } +} + +#[async_trait] +impl EmitMetrics for StaticGaugeWrapper { + async fn emit_metrics(&self, metrics: &SharedMetrics) { + self.inner() + .snapshot() + .emit::(metrics, |builder| builder); + } +} diff --git a/osprey_coordinator/src/metrics/histograms/dynamic_histogram.rs b/osprey_coordinator/src/metrics/histograms/dynamic_histogram.rs new file mode 100644 index 0000000..4f3f089 --- /dev/null +++ b/osprey_coordinator/src/metrics/histograms/dynamic_histogram.rs @@ -0,0 +1,182 @@ +use async_trait::async_trait; +use fxhash::FxHashMap; +use parking_lot::RwLock; +use std::{collections::hash_map::Entry, fmt::Debug, time::Duration}; + +use crate::metrics::base_metric::{BaseMetric, DynamicTagKey, EmitMetrics}; +use crate::metrics::SharedMetrics; + +use super::pool_histograms::{HistogramMetrics, PoolHistograms}; + +pub type DynamicHistogramStorage = RwLock>; + +pub trait DynamicHistogram: BaseMetric { + type Key: DynamicTagKey; + + #[doc(hidden)] + fn storage(&self) -> &DynamicHistogramStorage; + + #[doc(hidden)] + #[inline(always)] + fn record_with_key(&self, key: Self::Key, duration: Duration) { + self.record_value_with_key(key, duration.as_millis() as _) + } + + #[doc(hidden)] + #[inline(always)] + fn record_value_with_key(&self, key: Self::Key, value: u64) { + let storage = self.storage(); + + // First, try the *fast* path, for a dynamic key pair we've seen before. + let locked = storage.read(); + if let Some(pool_histograms) = locked.get(&key) { + pool_histograms.record_value(value); + return; + } + drop(locked); + + // Second, try the *slow* path, which is that we have not discovered this dynamic key pair before, + // so we must insert it into the storage. + let mut locked = storage.write(); + match locked.entry(key) { + Entry::Occupied(ent) => { + ent.get().record_value(value); + } + Entry::Vacant(ent) => { + let pool_histograms = PoolHistograms::new(); + pool_histograms.record_value(value); + ent.insert(pool_histograms); + } + } + } +} + +/// A wrapper that disambiguates between static and dynamic histograms for the purpose +/// of the EmitMetrics auto trait impl. +#[derive(Default)] +pub struct DynamicHistogramWrapper(Histogram); + +impl std::ops::Deref for DynamicHistogramWrapper { + type Target = Histogram; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Debug for DynamicHistogramWrapper { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DynamicHistogram") + .field("name", &Histogram::NAME) + .field("static_tags", &Histogram::STATIC_TAGS) + .field("dynamic_tag_keys", &Histogram::Key::get_tag_keys()) + .field("num_distinct_keys", &self.storage().read().len()) + .finish() + } +} + +#[async_trait] +impl EmitMetrics for DynamicHistogramWrapper +where + Histogram: DynamicHistogram, +{ + async fn emit_metrics(&self, metrics: &SharedMetrics) { + let storage = self.storage(); + let locked = storage.read(); + for (key, value) in locked.iter() { + if let Some(histogram_metrics) = value.get_histogram_metrics() { + Self::emit_dynamic_histogram_metrics(key, histogram_metrics, metrics); + } + } + } +} + +impl DynamicHistogramWrapper +where + Histogram: DynamicHistogram, +{ + fn emit_dynamic_histogram_metrics( + dynamic_key: &Histogram::Key, + histogram_metrics: HistogramMetrics, + metrics: &SharedMetrics, + ) { + Self::emit_counter_metrics( + dynamic_key, + metrics, + format!("{}.count", Histogram::NAME).as_str(), + histogram_metrics.sample_count as i64, + ); + Self::emit_float_gauge_metrics( + dynamic_key, + metrics, + format!("{}.avg", Histogram::NAME).as_str(), + histogram_metrics.mean, + ); + Self::emit_gauge_metrics( + dynamic_key, + metrics, + format!("{}.median", Histogram::NAME).as_str(), + histogram_metrics.median, + ); + Self::emit_gauge_metrics( + dynamic_key, + metrics, + format!("{}.95percentile", Histogram::NAME).as_str(), + histogram_metrics.p95, + ); + Self::emit_gauge_metrics( + dynamic_key, + metrics, + format!("{}.99percentile", Histogram::NAME).as_str(), + histogram_metrics.p99, + ); + Self::emit_gauge_metrics( + dynamic_key, + metrics, + format!("{}.min", Histogram::NAME).as_str(), + histogram_metrics.min, + ); + Self::emit_gauge_metrics( + dynamic_key, + metrics, + format!("{}.max", Histogram::NAME).as_str(), + histogram_metrics.max, + ); + } + + fn emit_counter_metrics( + dynamic_key: &Histogram::Key, + metrics: &SharedMetrics, + key: &str, + value: i64, + ) { + let mut builder = metrics.count_with_tags(key, value); + builder = Histogram::fold_static_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder = dynamic_key.fold_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder.try_send().ok(); + } + + fn emit_gauge_metrics( + dynamic_key: &Histogram::Key, + metrics: &SharedMetrics, + key: &str, + value: u64, + ) { + let mut builder = metrics.gauge_with_tags(key, value); + builder = Histogram::fold_static_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder = dynamic_key.fold_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder.try_send().ok(); + } + + fn emit_float_gauge_metrics( + dynamic_key: &Histogram::Key, + metrics: &SharedMetrics, + key: &str, + value: f64, + ) { + let mut builder = metrics.gauge_with_tags(key, value); + builder = Histogram::fold_static_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder = dynamic_key.fold_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder.try_send().ok(); + } +} diff --git a/osprey_coordinator/src/metrics/histograms/mod.rs b/osprey_coordinator/src/metrics/histograms/mod.rs new file mode 100644 index 0000000..4b6d640 --- /dev/null +++ b/osprey_coordinator/src/metrics/histograms/mod.rs @@ -0,0 +1,7 @@ +pub mod dynamic_histogram; +mod pool_histograms; +pub mod static_histogram; + +pub use dynamic_histogram::{DynamicHistogram, DynamicHistogramStorage}; +pub use pool_histograms::PoolHistograms; +pub use static_histogram::StaticHistogram; diff --git a/osprey_coordinator/src/metrics/histograms/pool_histograms.rs b/osprey_coordinator/src/metrics/histograms/pool_histograms.rs new file mode 100644 index 0000000..6e25ca0 --- /dev/null +++ b/osprey_coordinator/src/metrics/histograms/pool_histograms.rs @@ -0,0 +1,159 @@ +use std::{sync::Arc, thread, time::Duration}; + +use arc_swap::ArcSwap; +use dynamic_pool::{DynamicPool, DynamicReset}; +use hdrhistogram::Histogram; +use lazy_static::lazy_static; +use tracing::{error, warn}; + +lazy_static! { + static ref CPU_COUNT: usize = num_cpus::get(); +} + +/// A hdrHistogram wrapper with the DynamicReset trait implementation. +/// So this struct can be used as a DynamicPoolItem in a DynamicPool. +#[derive(Debug)] +pub struct PoolItemHistogram(Histogram); + +impl DynamicReset for PoolItemHistogram { + fn reset(&mut self) {} +} + +impl std::ops::Deref for PoolItemHistogram { + type Target = Histogram; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for PoolItemHistogram { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// A arc swappable hdrHistograms stored in a dynamic pool. +#[derive(Debug)] +pub struct PoolHistograms(ArcSwap>); + +impl std::ops::Deref for PoolHistograms { + type Target = ArcSwap>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Default for PoolHistograms { + fn default() -> Self { + Self::new() + } +} + +impl PoolHistograms { + pub(crate) fn new() -> Self { + PoolHistograms(ArcSwap::from(Self::create())) + } + + #[inline(always)] + fn create() -> Arc> { + Arc::new(DynamicPool::new(*CPU_COUNT, *CPU_COUNT, || { + // Create a Histogram that can count values in the [1..u64::max_value()] range with 1% precision. + // If the provided parameters to create this histogram are invalid, returns an error, which should not happen. + let h = hdrhistogram::Histogram::::new_with_bounds(1, u64::max_value(), 2) + .map_err(|e| { + error!["Couldn't create histogram: {:?}", e]; + e + }) + .unwrap(); + PoolItemHistogram(h) + })) + } + + #[inline(always)] + pub(crate) fn record(&self, duration: Duration) { + self.record_value(duration.as_millis() as _) + } + + #[inline(always)] + pub(crate) fn record_value(&self, value: u64) { + for _i in 0..2 { + if let Some(mut item) = self.load().try_take() { + if let Err(e) = item.record(value) { + warn!["Couldn't record value: {:?}", e]; + } + return; + } + + thread::yield_now(); + } + } + + #[inline(always)] + fn renew_and_swap(&self) -> Arc> { + self.swap(Self::create()) + } + + // Merge all the histograms in the dynamic pool and calcualte the histogram + pub fn get_histogram_metrics(&self) -> Option { + let mut remaining = *CPU_COUNT; + let old_histograms = self.renew_and_swap(); + let first = old_histograms.try_take()?; + let mut merged = first.detach().0; + remaining -= 1; + + for i in 0..2 { + while let Some(next) = old_histograms.try_take() { + let histogram = next.detach().0; + remaining -= 1; + merged.add(histogram).ok(); + } + + if remaining == 0 { + break; + } + thread::yield_now(); + + if i == 1 { + warn!( + "Couldn't take all items. Still remaining {} items in the dynamic pool.", + remaining + ); + } + } + + let sample_count = merged.len(); + if sample_count == 0 { + return None; + } + + let mean = merged.mean(); + let p95 = merged.value_at_quantile(0.95); + let p99 = merged.value_at_quantile(0.99); + let min = merged.min(); + let max = merged.max(); + let median = merged.value_at_quantile(0.5); + + Some(HistogramMetrics { + sample_count, + mean, + min, + max, + p95, + p99, + median, + }) + } +} + +#[derive(Debug)] +pub struct HistogramMetrics { + pub sample_count: u64, + pub mean: f64, + pub min: u64, + pub max: u64, + pub p95: u64, + pub p99: u64, + pub median: u64, +} diff --git a/osprey_coordinator/src/metrics/histograms/static_histogram.rs b/osprey_coordinator/src/metrics/histograms/static_histogram.rs new file mode 100644 index 0000000..f095242 --- /dev/null +++ b/osprey_coordinator/src/metrics/histograms/static_histogram.rs @@ -0,0 +1,115 @@ +use std::{fmt::Debug, time::Duration}; + +use crate::metrics::{BaseMetric, EmitMetrics, SharedMetrics}; +use async_trait::async_trait; + +use super::pool_histograms::{HistogramMetrics, PoolHistograms}; + +#[async_trait] +pub trait StaticHistogram: BaseMetric { + fn value(&self) -> &PoolHistograms; + + fn record(&self, duration: Duration) { + self.value().record(duration); + } + + fn record_value(&self, value: u64) { + self.value().record_value(value); + } +} + +/// A wrapper that disambiguates between static and dynamic histograms for the purpose +/// of the EmitMetrics auto trait impl. +#[derive(Default)] +pub struct StaticHistogramWrapper(Histogram); + +impl std::ops::Deref for StaticHistogramWrapper { + type Target = Histogram; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Debug for StaticHistogramWrapper { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StaticHistogram") + .field("name", &Histogram::NAME) + .field("static_tags", &Histogram::STATIC_TAGS) + .field("used_histograms", &self.value().load().used()) + .finish() + } +} + +#[async_trait] +impl EmitMetrics for StaticHistogramWrapper +where + Histogram: StaticHistogram, +{ + async fn emit_metrics(&self, metrics: &SharedMetrics) { + if let Some(histogram_metrics) = self.value().get_histogram_metrics() { + Self::emit_static_histogram_metrics(histogram_metrics, metrics); + } + } +} + +impl StaticHistogramWrapper +where + Histogram: StaticHistogram, +{ + fn emit_static_histogram_metrics(histogram_metrics: HistogramMetrics, metrics: &SharedMetrics) { + Self::emit_counter_metrics( + metrics, + format!("{}.count", Histogram::NAME).as_str(), + histogram_metrics.sample_count as i64, + ); + Self::emit_float_gauge_metrics( + metrics, + format!("{}.avg", Histogram::NAME).as_str(), + histogram_metrics.mean, + ); + Self::emit_gauge_metrics( + metrics, + format!("{}.median", Histogram::NAME).as_str(), + histogram_metrics.median, + ); + Self::emit_gauge_metrics( + metrics, + format!("{}.95percentile", Histogram::NAME).as_str(), + histogram_metrics.p95, + ); + Self::emit_gauge_metrics( + metrics, + format!("{}.99percentile", Histogram::NAME).as_str(), + histogram_metrics.p99, + ); + Self::emit_gauge_metrics( + metrics, + format!("{}.min", Histogram::NAME).as_str(), + histogram_metrics.min, + ); + Self::emit_gauge_metrics( + metrics, + format!("{}.max", Histogram::NAME).as_str(), + histogram_metrics.max, + ); + } + + fn emit_counter_metrics(metrics: &SharedMetrics, key: &str, value: i64) { + let mut builder = metrics.count_with_tags(key, value); + builder = Histogram::fold_static_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder.try_send().ok(); + } + + fn emit_gauge_metrics(metrics: &SharedMetrics, key: &str, value: u64) { + let mut builder = metrics.gauge_with_tags(key, value); + builder = Histogram::fold_static_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder.try_send().ok(); + } + + fn emit_float_gauge_metrics(metrics: &SharedMetrics, key: &str, value: f64) { + let mut builder = metrics.gauge_with_tags(key, value); + builder = Histogram::fold_static_tag_pairs(builder, |b, k, v| b.with_tag(k, v)); + builder.try_send().ok(); + } +} diff --git a/osprey_coordinator/src/metrics/macros/dynamic_metric.rs b/osprey_coordinator/src/metrics/macros/dynamic_metric.rs new file mode 100644 index 0000000..68f6cd0 --- /dev/null +++ b/osprey_coordinator/src/metrics/macros/dynamic_metric.rs @@ -0,0 +1,476 @@ +#[doc(hidden)] +#[macro_export] +macro_rules! __dynamic_metric_generate_struct_and_impls { + ($metric_type: ident, $metric_struct_name: ident, $($metric_args: tt)*) => { + $crate::__dynamic_metric_generate_struct_and_impls!( + @parse, + $metric_type, + $metric_struct_name, + [$($metric_args)*] + ); + }; + + // Phase 1: Parse AST + // + // This parse handles parsing of the DynamicCounter(...) variants. + // + // Variant 1: DynamicCounter([dynamic_tags, here => "..."]): Implicit name, dynamic tags provided. + (@parse, $metric_type: ident, $metric_struct_name: ident, [[$($dynamic_args: tt)+]]) => { + $crate::__dynamic_metric_generate_struct_and_impls!( + @parse_dynamic_init, + $metric_type, + [$($dynamic_args)+], + [ + metric_name: stringify!($metric_struct_name), + metric_struct_name: $metric_struct_name, + static_tags: [], + ] + ); + }; + // Variant 2: DynamicCounter([dynamic_tags, here => "..."], ["foo" => "bar"]): Implicit name, dynamic + static tags provided. + (@parse, $metric_type: ident, $metric_struct_name: ident, + [[$($dynamic_args: tt)+], + [$($static_tag_key: expr => $static_tag_value: expr),+]]) => { + $crate::__dynamic_metric_generate_struct_and_impls!( + @parse_dynamic_init, + $metric_type, + [$($dynamic_args)+], + [ + metric_name: stringify!($metric_struct_name), + metric_struct_name: $metric_struct_name, + static_tags: [$($static_tag_key => $static_tag_value,)+], + ] + ); + }; + // Variant 3: DynamicCounter("explicit.name", [dynamic_tags, here => "..."]): Explicit name + dynamic tags provided. + (@parse, $metric_type: ident, $metric_struct_name: ident, [$metric_name: expr, [$($dynamic_args: tt)+]]) => { + $crate::__dynamic_metric_generate_struct_and_impls!( + @parse_dynamic_init, + $metric_type, + [$($dynamic_args)+], + [ + metric_name: $metric_name, + metric_struct_name: $metric_struct_name, + static_tags: [], + ] + ); + }; + // Variant 4: DynamicCounter("explicit.name", [dynamic_tags, here => "..."], ["static_tags" => "are_here"]): Explicit name + dynamic +static tags provided. + (@parse, $metric_type: ident, $metric_struct_name: ident, + [$metric_name: expr, [$($dynamic_args: tt)+], + [$($static_tag_key: expr => $static_tag_value: expr),+]]) => { + $crate::__dynamic_metric_generate_struct_and_impls!( + @parse_dynamic_init, + $metric_type, + [$($dynamic_args)+], + [ + metric_name: $metric_name, + metric_struct_name: $metric_struct_name, + static_tags: [$($static_tag_key => $static_tag_value,)+], + ] + ); + }; + + // Phase 2: Parse Dynamic Init + // + // These macro heads will try to parse the dynamic token tags, normalizing the following: + // a) [foo, bar] ====> [foo => "foo", bar => "bar"] + // b) [foo, bar => "bar.aliased"] ====> [foo => "foo", bar => "bar.aliased"] + // This allows us to accept dynamic tags that have an ident, and an alias to a string, if necessary. + // This step just sets up the accumulators, and then advance to phase 3. + (@parse_dynamic_init, $metric_type: ident, [$($dynamic_args: tt)*], [$($config: tt)*]) => { + $crate::__dynamic_metric_generate_struct_and_impls!( + @parse_dynamic, + $metric_type, + [$($dynamic_args)*], + [], + [$($config)*] + ); + }; + + // Phase 3: Parse Dynamic + // + // Munch an ident with a comma e.g. `foo,` + (@parse_dynamic, $metric_type: ident, [$dynamic_arg_name: ident, $($dynamic_args_rest: tt)+], [$($acc: tt)*], [$($config: tt)*]) => { + $crate::__dynamic_metric_generate_struct_and_impls!( + @parse_dynamic, + $metric_type, + [$($dynamic_args_rest)+], + [$($acc)* $dynamic_arg_name => stringify!($dynamic_arg_name),], + [$($config)*] + ); + }; + // Munch an ident without a trailing comma, at the end of the tag list, e.g. `foo` + (@parse_dynamic, $metric_type: ident, [$dynamic_arg_name: ident], [$($acc: tt)*], [$($config: tt)*]) => { + $crate::__dynamic_metric_generate_struct_and_impls!( + @parse_dynamic, + $metric_type, + [], + [$($acc)* $dynamic_arg_name => stringify!($dynamic_arg_name)], + [$($config)*] + ); + }; + + // Munch an ident => expr with a comma e.g. `foo => "foo.aliased",` + (@parse_dynamic, $metric_type: ident, + [$dynamic_arg_name: ident => $dynamic_tag_key: expr, $($dynamic_args_rest: tt)+], + [$($acc: tt)*], [$($config: tt)*]) => { + $crate::__dynamic_metric_generate_struct_and_impls!( + @parse_dynamic, + $metric_type, + [$($dynamic_args_rest)+], + [$($acc)* $dynamic_arg_name => $dynamic_tag_key,], + [$($config)*] + ); + }; + + // Munch an ident => expr without a trailing comma, at the end of the tag list, e.g. `foo => "foo.aliased"` + (@parse_dynamic, $metric_type: ident, [$dynamic_arg_name: ident => $dynamic_tag_key: expr], + [$($acc: tt)*], [$($config: tt)*]) => { + $crate::__dynamic_metric_generate_struct_and_impls!( + @parse_dynamic, + $metric_type, + [], + [$($acc)* $dynamic_arg_name => $dynamic_tag_key], + [$($config)*] + ); + }; + // Nothing more to parse, move onto phase 4. Phase 4 has variants for DynamicCounter and DynamicHistogram + (@parse_dynamic, $metric_type: ident, [], + [$($dynamic_arg_name: ident => $dynamic_tag_key: expr),+], [$($config: tt)*]) => { + $crate::__dynamic_metric_generate_struct_and_impls!( + @gen, $metric_type, [ + $($config)* + dynamic_args: [ + $($dynamic_arg_name => $dynamic_tag_key,)+ + ], + ] + ); + }; + // Some shared code gen needed in phase 4. + (@gen_shared, $metric_name: expr, + $metric_struct_name: ident, + [$($static_tag_key: expr => $static_tag_value: expr,)*], + [$($dynamic_tag_field_name: ident => $dynamic_tag_key: expr,)+], + ) => { + $crate::metrics::macros::paste! { + #[derive(Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] + pub struct [<$metric_struct_name:camel Key>] { + $(pub $dynamic_tag_field_name: &'static str,)+ + } + + impl prelude::DynamicTagKey for [<$metric_struct_name:camel Key>] { + fn fold_tag_pairs(&self, mut t: T, f: F) -> T + where + F: Fn(T, &'static str, &'static str) -> T, + { + $(t = f(t, $dynamic_tag_key, self.$dynamic_tag_field_name);)+ + t + } + + fn get_tag_keys() -> &'static [&'static str] { + &[$($dynamic_tag_key,)+] + } + } + + impl prelude::BaseMetric for [<$metric_struct_name:camel>] { + const NAME: &'static str = $metric_name; + const STATIC_TAGS: &'static [prelude::StaticTag] = &[ + $( + prelude::StaticTag::new($static_tag_key, $static_tag_value), + )* + ]; + } + } + }; + // Phase 4: Code Gen for DynamicCounter + // + // Given the transformed AST that now has all the data we need to emit code, perform + // code-gen here. + (@gen, DynamicCounter, [ + metric_name: $metric_name: expr, + metric_struct_name: $metric_struct_name: ident, + static_tags: [ + $($static_tag_key: expr => $static_tag_value: expr,)* + ], + dynamic_args: [ + $($dynamic_tag_field_name: ident => $dynamic_tag_key: expr,)+ + ], + ]) => { + $crate::metrics::macros::paste! { + pub struct [<$metric_struct_name:camel>] { + storage: prelude::DynamicCounterStorage<[<$metric_struct_name:camel Key>]>, + } + + $crate::__dynamic_metric_generate_struct_and_impls!(@gen_shared, + $metric_name, + $metric_struct_name, + [$($static_tag_key => $static_tag_value,)*], + [$($dynamic_tag_field_name => $dynamic_tag_key,)+], + ); + + impl prelude::DynamicCounter for [<$metric_struct_name:camel>] { + type Key = [<$metric_struct_name:camel Key>]; + + fn storage(&self) -> &prelude::DynamicCounterStorage { + &self.storage + } + } + + #[allow(clippy::too_many_arguments)] + impl [<$metric_struct_name:camel>] { + #[allow(unused)] + pub fn incr(&self, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+) { + prelude::DynamicCounter::incr_with_key( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + 1 + ); + } + + #[allow(unused)] + pub fn incr_by(&self, by: i64, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+) { + prelude::DynamicCounter::incr_with_key( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + by + ); + } + + #[allow(unused)] + pub fn decr(&self, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+) { + prelude::DynamicCounter::decr_with_key( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + 1 + ); + } + + #[allow(unused)] + pub fn decr_by(&self, by: i64, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+) { + prelude::DynamicCounter::decr_with_key( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + by + ); + } + } + + impl Default for [<$metric_struct_name:camel>] { + fn default() -> Self { + Self { + storage: Default::default() + } + } + } + } + }; + // Phase 4: Code Gen for DynamicGauge + // + // Given the transformed AST that now has all the data we need to emit code, perform + // code-gen here. + (@gen, DynamicGauge, [ + metric_name: $metric_name: expr, + metric_struct_name: $metric_struct_name: ident, + static_tags: [ + $($static_tag_key: expr => $static_tag_value: expr,)* + ], + dynamic_args: [ + $($dynamic_tag_field_name: ident => $dynamic_tag_key: expr,)+ + ], + ]) => { + $crate::metrics::macros::paste! { + pub struct [<$metric_struct_name:camel>] { + storage: prelude::DynamicGaugeStorage<[<$metric_struct_name:camel Key>]>, + } + + $crate::__dynamic_metric_generate_struct_and_impls!(@gen_shared, + $metric_name, + $metric_struct_name, + [$($static_tag_key => $static_tag_value,)*], + [$($dynamic_tag_field_name => $dynamic_tag_key,)+], + ); + + impl prelude::DynamicGauge for [<$metric_struct_name:camel>] { + type Key = [<$metric_struct_name:camel Key>]; + + fn storage(&self) -> &prelude::DynamicGaugeStorage { + &self.storage + } + } + + #[allow(clippy::too_many_arguments)] + impl [<$metric_struct_name:camel>] { + /// Increments the gauge by the given value with metrics tags. + /// + /// # Safety + /// This method must be called with an accompanied [`DyanmicGauge::decr_by`], otherwise the state of the gauge + /// will be incorrect. + /// + /// More conveniently and safely though, the use of [`DyanmicGauge::acquire_owned`] + /// or [`DynamicGauge::acquire_many_owned`] is recommended, + /// which automatically decrements the gauge for you when the returned [`GaugeHandleOwned`] falls out of scope. + #[allow(unused)] + pub unsafe fn incr_by(&self, by: u64, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+) { + prelude::DynamicGauge::incr_with_key( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + by + ); + } + + /// Increments the gauge by 1. + /// + /// # Safety + /// This method must be called with an accompanied [`DyanmicGauge::decr`], otherwise the state of the gauge + /// will be incorrect. + /// + /// See [`DynamicGauge::incr_by`] for more information. + #[allow(unused)] + pub unsafe fn incr(&self, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+) { + prelude::DynamicGauge::incr_with_key( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + 1 + ); + } + + /// Decrements the gauge by 1. + /// + /// # Safety + /// The gauge value after decrement could underflow. + /// + /// See [`StaticGauge::incr_by`] for more information. + #[allow(unused)] + pub unsafe fn decr(&self, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+) { + prelude::DynamicGauge::decr_with_key( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + 1 + ); + } + + /// Decrements the gauge by a given value. + /// + /// # Safety + /// The gauge value after decrement could underflow. + /// + /// See [`StaticGauge::incr_by`] for more information. + #[allow(unused)] + pub unsafe fn decr_by(&self, by: u64, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+) { + prelude::DynamicGauge::decr_with_key( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + by + ); + } + + /// Sets the gauge to a given value with metrics tags. + /// This method is useful when you have a background task that periodically reports a given value. + #[allow(unused)] + pub fn set(&self, val: u64, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+) { + prelude::DynamicGauge::set_with_key( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + val + ); + } + + /// Increments the gauge by a given value with metrics tags + /// and returns a [`GaugeHandleOwned`] which will automatically decrement the gauge by that value when dropped. + #[allow(unused)] + pub fn acquire_many_owned(&self, val: u64, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+) + -> prelude::GaugeHandleOwned { + prelude::DynamicGauge::acquire_many_with_key_owned( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + val + ) + } + + /// Increments the gauge by 1 with metrics tags + /// and returns a [`GaugeHandleOwned`] which will automatically decrement the gauge when dropped. + #[allow(unused)] + pub fn acquire_owned(&self, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+) + -> prelude::GaugeHandleOwned { + prelude::DynamicGauge::acquire_with_key_owned( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ } + ) + } + } + + impl Default for [<$metric_struct_name:camel>] { + fn default() -> Self { + Self { + storage: Default::default() + } + } + } + } + }; + // Phase 4: Code Gen for DynamicHistogram + // + // Given the transformed AST that now has all the data we need to emit code, perform + // code-gen here. + (@gen, DynamicHistogram, [ + metric_name: $metric_name: expr, + metric_struct_name: $metric_struct_name: ident, + static_tags: [ + $($static_tag_key: expr => $static_tag_value: expr,)* + ], + dynamic_args: [ + $($dynamic_tag_field_name: ident => $dynamic_tag_key: expr,)+ + ], + ]) => { + $crate::metrics::macros::paste! { + pub struct [<$metric_struct_name:camel>] { + storage: prelude::DynamicHistogramStorage<[<$metric_struct_name:camel Key>]>, + } + + $crate::__dynamic_metric_generate_struct_and_impls!(@gen_shared, + $metric_name, + $metric_struct_name, + [$($static_tag_key => $static_tag_value,)*], + [$($dynamic_tag_field_name => $dynamic_tag_key,)+], + ); + + impl prelude::DynamicHistogram for [<$metric_struct_name:camel>] { + type Key = [<$metric_struct_name:camel Key>]; + + fn storage(&self) -> &prelude::DynamicHistogramStorage { + &self.storage + } + } + + #[allow(clippy::too_many_arguments)] + impl [<$metric_struct_name:camel>] { + #[allow(unused)] + pub fn record_with_key(&self, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+, duration: std::time::Duration) { + prelude::DynamicHistogram::record_with_key( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + duration + ); + } + + #[allow(unused)] + pub fn record_value_with_key(&self, $($dynamic_tag_field_name: impl prelude::DynamicTagValue),+, value: u64) { + prelude::DynamicHistogram::record_value_with_key( + self, + [<$metric_struct_name:camel Key>] { $($dynamic_tag_field_name: $dynamic_tag_field_name.as_static_str()),+ }, + value + ); + } + } + + impl Default for [<$metric_struct_name:camel>] { + fn default() -> Self { + Self { + storage: Default::default() + } + } + } + } + }; +} diff --git a/osprey_coordinator/src/metrics/macros/mod.rs b/osprey_coordinator/src/metrics/macros/mod.rs new file mode 100644 index 0000000..c909ef3 --- /dev/null +++ b/osprey_coordinator/src/metrics/macros/mod.rs @@ -0,0 +1,283 @@ +mod dynamic_metric; +mod static_metric; + +pub use async_trait::async_trait; +pub use paste::paste; + +/// Defines a set of metrics that can be collected for a given application. +/// +/// The `define_metrics!` macro should be used at the module level, and not within +/// a function. +/// +/// The `define_metrics!` macro accepts the following parts: +/// 1) `MetricStructName` +/// 2) A list of *metrics*, and their configuration. +/// +/// ### The following *metrics* are currently supported. +/// +/// - `StaticCounter`: A very fast static counter that can be used in super high-throughput codepaths. +/// The underlying counter is implemented using a single [`std::sync::atomic::AtomicI64`], and thus +/// is very fast. +/// - `DynamicCounter`: A counter that allows for dynamic "tags" to be applied. These tags translate to +/// to statsd tags, and can be used in places where the tags need to be programtically computed at +/// runtime. Compared to a static counter, dynamic counters are much slower, as they involve a RwLock +/// and a hashmap. The RwLock is only acquired in write mode for a new dynamic tag combination that has +/// not been seen before in the runtime of the program. Additionally, dynamic counters can only take +/// tags which are `&'static str`'s, guaranteeing that the set of tags will not grow in an unbounded +/// fashion during runtime (assuming you aren't Box::leak'ing Strings). +/// - `StaticGauge`: A very fast static gauge that can be used in super high-throughput codepaths. +/// The underlying gauge is implemented using three single [`std::sync::atomic::AtomicU64`] to track +/// the gauge current value and min, max values in the reporting window, and thus is very fast. +/// - `DynamicGauge`: A gauge that allows for dynamic "tags" to be applied. The underlying implementation +/// of DyanmicGauge also involves a RwLock and a hashmap. Therefore, it is much slower than static gauges. +/// - `StaticHistogram`: A very fast static histogram that can be used in super high-throughput codepaths. +/// The underlying implementation uses many [`hdrhistogram::Histogram`] whose maximum count is +/// the number of cpus on this node. These histograms are stored in a thread safe [`dynamic_pool::DynamicPool`]. +/// - `DynamicHistogram`: A histogram that allows for dynamic "tags" to be applied. The underlying implementation +/// of DyanmicHistogram also involves a RwLock and a hashmap. Each histogram per dynamic "tag" is the same as +/// the thread safe pool of histograms used in `StaticHistogram`. +/// +/// +/// More details on the metric types can be found below. +/// The macro definition of Histogram and Gauge is the same as Counter. +/// +/// ### Syntax for defining metrics. +/// +/// The syntax for defining a metric is as follows: +/// +/// ```no_run +/// use crate::metrics::define_metrics; +/// +/// define_metrics!(FooMetrics, [ +/// foo => StaticCounter(), +/// bar => DynamicCounter([tag_a, tag_b]), +/// ]); +/// ``` +/// +/// To break this down, `FooMetrics` is the name of the struct that is generated that holds all the metrics +/// defined. `foo`, and `bar` are fields within the struct that contain the metric to the right hand-side of +/// the arrow `=>`. +/// +/// In order to use the metrics, you must first create an instance of `FooMetrics`, usually this is done by using +/// the `new` method, which will return an `Arc`. +/// ``` +/// use crate::metrics::define_metrics; +/// use crate::metrics::counters::StaticCounter; +/// use crate::metrics::counters::DynamicCounter; +/// +/// define_metrics!(FooMetrics, [ +/// foo => StaticCounter(), +/// bar => DynamicCounter([tag_a, tag_b]), +/// ]); +/// fn main() { +/// let metrics = FooMetrics::new(); +/// metrics.foo.incr(); +/// metrics.bar.incr("tag_a_value", "tag_b_value"); +/// } +/// ``` +/// +/// Besides counting your metrics, however, these metrics will not be emitted anywhere. This is because you need +/// to spawn a background worker to flush the metrics to a [`crate::SharedMetrics`]. The odd naming right now is +/// due to a transitional period where we are both using the legacy [`crate::SharedMetrics`] to emit metrics, and +/// also as the underlying statsd client which is used to emit metrics. +/// +/// In order to spawn metrics, we simply need to employ the [`crate::SpawnEmitWorker`] trait. This must be ran within +/// a tokio runtime context. Putting this all together: +/// +/// ```no_run +/// use crate::metrics::{new_client, define_metrics, SpawnEmitWorker}; +/// use crate::metrics::counters::StaticCounter; +/// +/// define_metrics!(FooMetrics, [ +/// foo => StaticCounter(), +/// bar => DynamicCounter([tag_a, tag_b]), +/// ]); +/// +/// #[tokio::main] +/// async fn main() { +/// let metrics = FooMetrics::new(); +/// let _worker_guard = metrics.clone().spawn_emit_worker(new_client("prefix.here").unwrap()); +/// +/// metrics.foo.incr(); +/// metrics.bar.incr("a", "b"); +/// } +/// ``` +/// +/// ## Metric Types: +/// +/// ### `StaticCounter` +/// +/// A static counter can be defined in the following forms: +/// ``` +/// # use crate::metrics::define_metrics; +/// define_metrics!(FooMetrics, [ +/// foo => StaticCounter(), +/// bar => StaticCounter("explicit.name"), +/// baz => StaticCounter(["static" => "tags", "are" => "defined.here"]), +/// qux => StaticCounter("explicit.name", ["static" => "tags"]), +/// ]); +/// ``` +/// +/// You can think of this as the following: `StaticCounter("explicit.name", ["static" => "tags"])`, where +/// the explicit name and static tags arguments are optional. +/// +/// If an explicit name is not provided, the field name within the `define_metrics!` macro +/// (e.g. `foo` in `foo => StaticCounter()`) will be used as the metric name. +/// +/// Static tags are defined as a list of `key => value` strings. For example: `["static" => "tags", "are" => "defined.here"]`. +/// +/// Providing tags, or an explicit name will not increase the runtime overhead. +/// +/// ### `DynamicCounter` +/// +/// A dynamic counter can be defined in the following forms: +/// ``` +/// # use crate::metrics::define_metrics; +/// define_metrics!(FooMetrics, [ +/// foo => DynamicCounter([tag_a, tag_b]), +/// bar => DynamicCounter("explicit.name", [tag_a, tag_b]), +/// baz => DynamicCounter("explicit.name", [tag_a, tag_b], ["static" => "tags"]), +/// ]); +/// ``` +/// +/// You can think of this this as the following: `DynamicCounter("explicit.name", [tag_a, tag_b], ["static" => "tags"])` +/// where the following parts are optional: `"explicit.name"` and `["static" => "tags"]`. +/// +/// If an explicit name is not provided, the field name within the `define_metrics!` macro +/// (e.g. `foo` in `foo => DynamicCounter([tag_a])`) will be used as the metric name. +/// +/// Static tags are defined as a list of `key => value` strings. For example: `["static" => "tags", "are" => "defined.here"]`. +/// +/// Dynamic tags are defined as a list of ident's, e.g. `[foo, bar, baz]`. +/// +/// Additionally, dynamic tags can be aliased, allowing you to emit metrics which are not valid rust ident's, for example: +/// ``` +/// # use crate::metrics::define_metrics; +/// define_metrics!(FooMetrics, [ +/// foo => DynamicCounter([tag_a => "tag.a", tag_b => "tag.b"]), +/// ]); +/// ``` +/// +/// So we can define the dynamic tags as: `[foo]`, `[foo => "alias"]`, `[foo, bar => "bar.alias"]`. Where the ident is required, +/// and an optional `=> "alias"` can be provided after each ident. +/// +/// Providing static tags, or an explicit name will not increase the runtime overhead. +/// +#[macro_export] +macro_rules! define_metrics { + ($metrics_struct_name: ident, [ + $($metric_name: ident => $metric_type:ident($($metric_args: tt)*),)+ + ]) => { + #[derive(Debug, Default)] + #[allow(non_camel_case_types)] + pub struct $metrics_struct_name { + $(pub(crate) $metric_name: $crate::__generate_metric_struct_field_type!($metric_type, $metrics_struct_name, $metric_name)),+ + } + + impl $metrics_struct_name { + #[allow(unused)] + pub fn new() -> ::std::sync::Arc { + Default::default() + } + } + + #[$crate::metrics::macros::async_trait] + impl $crate::metrics::EmitMetrics for $metrics_struct_name { + async fn emit_metrics(&self, metrics: &$crate::metrics::SharedMetrics) { + $( + self.$metric_name.emit_metrics(metrics).await; + )+ + } + } + + $crate::metrics::macros::paste! { + mod [<$metrics_struct_name:snake:lower>] { + mod prelude { + #[allow(unused_imports)] + pub use crate::metrics::{BaseMetric, DynamicTagKey, DynamicTagValue, StaticTag}; + #[allow(unused_imports)] + pub use std::sync::atomic::AtomicI64; + #[allow(unused_imports)] + pub use crate::metrics::counters::{DynamicCounter, StaticCounter, DynamicCounterStorage}; + #[allow(unused_imports)] + pub use crate::metrics::gauges::{DynamicGauge, GaugeInner, StaticGauge, DynamicGaugeStorage}; + #[allow(unused_imports)] + pub use crate::metrics::gauges::{GaugeHandleOwned, Many, One}; + #[allow(unused_imports)] + pub use crate::metrics::histograms::{DynamicHistogram, StaticHistogram, DynamicHistogramStorage, PoolHistograms}; + } + + $( + $crate::__generate_metric_struct_and_impls!($metric_type, $metric_name, $($metric_args)*); + )+ + } + } + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __generate_metric_struct_field_type { + (DynamicCounter, $metrics_module_name: ident, $metric_struct_name: ident) => { + $crate::metrics::macros::paste! { + crate::metrics::counters::dynamic_counter::DynamicCounterWrapper<[<$metrics_module_name:snake:lower>]::[<$metric_struct_name:camel>]> + } + }; + (StaticCounter, $metrics_module_name: ident, $metric_struct_name: ident) => { + $crate::metrics::macros::paste! { + crate::metrics::counters::static_counter::StaticCounterWrapper<[<$metrics_module_name:snake:lower>]::[<$metric_struct_name:camel>]> + } + }; + (DynamicGauge, $metrics_module_name: ident, $metric_struct_name: ident) => { + $crate::metrics::macros::paste! { + crate::metrics::gauges::dynamic_gauge::DynamicGaugeWrapper<[<$metrics_module_name:snake:lower>]::[<$metric_struct_name:camel>]> + } + }; + (StaticGauge, $metrics_module_name: ident, $metric_struct_name: ident) => { + $crate::metrics::macros::paste! { + crate::metrics::gauges::static_gauge::StaticGaugeWrapper<[<$metrics_module_name:snake:lower>]::[<$metric_struct_name:camel>]> + } + }; + (DynamicHistogram, $metrics_module_name: ident, $metric_struct_name: ident) => { + $crate::metrics::macros::paste! { + crate::metrics::histograms::dynamic_histogram::DynamicHistogramWrapper<[<$metrics_module_name:snake:lower>]::[<$metric_struct_name:camel>]> + } + }; + (StaticHistogram, $metrics_module_name: ident, $metric_struct_name: ident) => { + $crate::metrics::macros::paste! { + crate::metrics::histograms::static_histogram::StaticHistogramWrapper<[<$metrics_module_name:snake:lower>]::[<$metric_struct_name:camel>]> + } + }; + ($other: ident, $metrics_module_name: ident, $metric_struct_name: ident) => { + compile_error!(concat!( + "Unknown metric type: ", stringify!($other), + ". Must be one of DynamicCounter, StaticCounter, DynamicGauge, StaticGauge, DynamicHistogram or StaticHistogram" + )) + } +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __generate_metric_struct_and_impls { + (StaticCounter, $metric_name: ident, $($metric_args: tt)*) => { + $crate::__static_metric_generate_struct_and_impls!(StaticCounter, $metric_name, AtomicI64, $($metric_args)*); + }; + (DynamicCounter, $metric_name: ident, $($metric_args: tt)*) => { + $crate::__dynamic_metric_generate_struct_and_impls!(DynamicCounter, $metric_name, $($metric_args)*); + }; + (StaticGauge, $metric_name: ident, $($metric_args: tt)*) => { + $crate::__static_metric_generate_struct_and_impls!(StaticGauge, $metric_name, GaugeInner, $($metric_args)*); + }; + (DynamicGauge, $metric_name: ident, $($metric_args: tt)*) => { + $crate::__dynamic_metric_generate_struct_and_impls!(DynamicGauge, $metric_name, $($metric_args)*); + }; + (StaticHistogram, $metric_name: ident, $($metric_args: tt)*) => { + $crate::__static_metric_generate_struct_and_impls!(StaticHistogram, $metric_name, PoolHistograms, $($metric_args)*); + }; + (DynamicHistogram, $metric_name: ident, $($metric_args: tt)*) => { + $crate::__dynamic_metric_generate_struct_and_impls!(DynamicHistogram, $metric_name, $($metric_args)*); + }; + ($other: ident, $metric_name: ident, $($metric_args: tt)*) => { + // Do nothing, the error is handled in `__generate_metric_struct_field_type`, let's not contribute to more error spam. + } +} diff --git a/osprey_coordinator/src/metrics/macros/static_metric.rs b/osprey_coordinator/src/metrics/macros/static_metric.rs new file mode 100644 index 0000000..9e5bb93 --- /dev/null +++ b/osprey_coordinator/src/metrics/macros/static_metric.rs @@ -0,0 +1,135 @@ +/// Handler for `StaticCounter(...), StaticGauge(...) or StaticHistogram(...)`. + +#[doc(hidden)] +#[macro_export] +macro_rules! __static_metric_generate_struct_and_impls { + // Generates the static metric struct + trait implementation based on parsing the metric args. + ($metric_type: ident, $metric_struct_name: ident, $value_type: ty, $($metric_args: tt)*) => { + $crate::__static_metric_generate_struct_and_impls!( + @parse, + $metric_type, + $metric_struct_name, + $value_type, + [$($metric_args)*] + ); + }; + + // Parse Args: + // This section handles parsing the arguments of `StaticCounter/StaticGauge/StaticHistogram`, namely the following variants: + // The document below will use StaticCounter as an example. + // + // 1: StaticCounter(): Tag name inferred, no static tags defined. + (@parse, $metric_type: ident, $metric_struct_name: ident, $value_type: ty, []) => { + $crate::__static_metric_generate_struct_and_impls!( + @gen, + $metric_type, + $metric_struct_name, + $value_type, + [ + metric_name: stringify!($metric_struct_name), + static_tags: [], + ] + ); + }; + // 2: StaticCounter(["static_tags" => "are_here"]): Tag name inferred, static tags defined. + (@parse, $metric_type: ident, $metric_struct_name: ident, $value_type: ty, [[$($tag_key:expr => $tag_value: expr),*]]) => { + $crate::__static_metric_generate_struct_and_impls!( + @gen, + $metric_type, + $metric_struct_name, + $value_type, + [ + metric_name: stringify!($metric_struct_name), + static_tags: [$($tag_key => $tag_value),*], + ] + ); + }; + // 3: StaticCounter("name.here"): Tag name explicit, no static tags defined. + (@parse, $metric_type: ident, $metric_struct_name: ident, $value_type: ty, [$metric_name: expr]) => { + $crate::__static_metric_generate_struct_and_impls!( + @gen, + $metric_type, + $metric_struct_name, + $value_type, + [ + metric_name: $metric_name, + static_tags: [], + ] + ); + }; + // 4: StaticCounter("name.here", ["static_tags" => "are_here"]): Tag name expicit, static tags defined. + (@parse, $metric_type: ident, $metric_struct_name: ident, $value_type: ty, + [$metric_name: expr, [$($tag_key:expr => $tag_value: expr),*]]) => { + $crate::__static_metric_generate_struct_and_impls!( + @gen, + $metric_type, + $metric_struct_name, + $value_type, + [ + metric_name: $metric_name, + static_tags: [$($tag_key => $tag_value),*], + ] + ); + }; + + // Following the successful result of the parser above, actually write the impl code for StaticGauge variant: + (@gen, StaticGauge, $metric_struct_name: ident, $value_type: ty, [ + metric_name: $metric_name: expr, + static_tags: [$($tag_key:expr => $tag_value: expr),*], + ]) => { + $crate::metrics::macros::paste! { + #[derive(Default)] + pub struct [<$metric_struct_name:camel>] { + value: std::sync::Arc, + } + + impl prelude::BaseMetric for [<$metric_struct_name:camel>] { + const NAME: &'static str = $metric_name; + const STATIC_TAGS: &'static [prelude::StaticTag] = &[ + $( + prelude::StaticTag::new($tag_key, $tag_value), + )* + ]; + } + + impl prelude::StaticGauge for [<$metric_struct_name:camel>] { + fn inner(&self) -> &prelude::$value_type { + &self.value.as_ref() + } + + fn inner_arc(&self) -> std::sync::Arc { + std::sync::Arc::clone(&self.value) + } + } + } + }; + + // Following the successful result of the parser above, actually write the impl code + // for StaticCounter and StaticHistogram variants: + (@gen, $metric_type: ident, $metric_struct_name: ident, $value_type: ty, [ + metric_name: $metric_name: expr, + static_tags: [$($tag_key:expr => $tag_value: expr),*], + ]) => { + $crate::metrics::macros::paste! { + #[derive(Default)] + pub struct [<$metric_struct_name:camel>] { + value: prelude::$value_type, + } + + impl prelude::BaseMetric for [<$metric_struct_name:camel>] { + const NAME: &'static str = $metric_name; + const STATIC_TAGS: &'static [prelude::StaticTag] = &[ + $( + prelude::StaticTag::new($tag_key, $tag_value), + )* + ]; + } + + impl prelude::$metric_type for [<$metric_struct_name:camel>] { + fn value(&self) -> &prelude::$value_type { + &self.value + } + } + } + } +} diff --git a/osprey_coordinator/src/metrics/metrics_derive/Cargo.toml b/osprey_coordinator/src/metrics/metrics_derive/Cargo.toml new file mode 100644 index 0000000..e69de29 diff --git a/osprey_coordinator/src/metrics/metrics_derive/src/lib.rs b/osprey_coordinator/src/metrics/metrics_derive/src/lib.rs new file mode 100644 index 0000000..e69de29 diff --git a/osprey_coordinator/src/metrics/mod.rs b/osprey_coordinator/src/metrics/mod.rs new file mode 100644 index 0000000..c6acd1c --- /dev/null +++ b/osprey_coordinator/src/metrics/mod.rs @@ -0,0 +1,203 @@ +pub mod base_metric; +pub mod counters; +pub mod emit_worker; +pub mod gauges; +pub mod histograms; +pub mod macros; +mod recycling_socket_metrics_sink; +pub mod string_interner; +pub mod v2; + +pub use crate::define_metrics; +pub use metrics_derive::{MetricsEmitter, TagKey}; + +use crate::metrics::recycling_socket_metrics_sink::BufferedUdpMetricSink; +pub use base_metric::{BaseMetric, DynamicTagKey, DynamicTagValue, EmitMetrics, StaticTag}; +use cadence::{ + ext::{ToCounterValue, ToGaugeValue, ToHistogramValue, ToTimerValue}, + Counted, Counter, Gauge, Gauged, Histogram, Histogrammed, MetricBuilder, MetricResult, + QueuingMetricSink, StatsdClient, Timed, Timer, +}; +use std::sync::Arc; +use std::{env, fmt::Debug}; +use thiserror::Error; + +pub type SharedMetrics = Arc; + +#[derive(Debug, Clone, Error)] +#[error("Failed to build metrics client: {0}")] +pub struct MetricsClientBuildError(String); + +pub fn new_client(prefix: impl Into) -> Result { + MetricsClientBuilder::new(prefix).build() +} + +#[derive(Debug, Clone)] +pub struct MetricsClientBuilder { + prefix: String, + tags: Vec<(String, String)>, +} + +impl MetricsClientBuilder { + pub fn new(prefix: impl Into) -> Self { + Self { + prefix: prefix.into(), + tags: Vec::new(), + } + } + + pub fn build(mut self) -> Result { + let host = std::env::var("DD_AGENT_HOST") + .or_else(|_| std::env::var("DATADOG_TRACE_AGENT_HOSTNAME")) + .unwrap_or_else(|_| "127.0.0.1".into()); + let host = (host.as_ref(), 8125); + let buffered_sink = BufferedUdpMetricSink::from(host) + .map_err(|e| MetricsClientBuildError(format!("metric error: {e}")))?; + let queuing_sink = QueuingMetricSink::from(buffered_sink); + + let mut tags = base_tags(); + tags.append(&mut self.tags); + + let wsc = WrappedStatsdClient::with_base_tags( + StatsdClient::from_sink(self.prefix.as_ref(), queuing_sink), + tags, + ); + + Ok(Arc::new(wsc)) + } + + pub fn with_tags(mut self, tags: Vec<(String, String)>) -> Self { + self.tags.extend(tags); + self + } + + pub fn with_tag(mut self, key: impl Into, value: impl Into) -> Self { + self.tags.push((key.into(), value.into())); + self + } +} + +fn base_tags() -> Vec<(String, String)> { + let mut tags = Vec::new(); + + if let Ok(tag) = env::var("DD_ENTITY_ID") { + tags.push(("dd.internal.entity_id".to_owned(), tag)) + } + + tags +} + +#[derive(Debug)] +pub struct WrappedStatsdClient { + base_tags: Vec<(String, String)>, + wrapped: StatsdClient, +} + +impl WrappedStatsdClient { + pub fn new(wrapped: StatsdClient) -> Self { + Self { + base_tags: vec![], + wrapped, + } + } + + pub fn with_base_tags(wrapped: StatsdClient, base_tags: I) -> Self + where + I: IntoIterator, + K: Into, + V: Into, + { + Self { + base_tags: base_tags + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(), + wrapped, + } + } + + // gauge + pub fn gauge_with_tags<'a, T: ToGaugeValue>( + &'a self, + key: &'a str, + value: T, + ) -> MetricBuilder<'a, 'a, Gauge> { + self.base_tags + .iter() + .fold(self.wrapped.gauge_with_tags(key, value), |acc, (k, v)| { + acc.with_tag(k, v) + }) + } + + pub fn gauge<'a, T: ToGaugeValue>(&'a self, key: &'a str, value: T) -> MetricResult { + self.gauge_with_tags(key, value).try_send() + } + + // time: + + pub fn time_with_tags<'a, T: ToTimerValue>( + &'a self, + key: &'a str, + value: T, + ) -> MetricBuilder<'a, 'a, Timer> { + self.base_tags + .iter() + .fold(self.wrapped.time_with_tags(key, value), |acc, (k, v)| { + acc.with_tag(k, v) + }) + } + + pub fn time<'a, T: ToTimerValue>(&'a self, key: &'a str, value: T) -> MetricResult { + self.time_with_tags(key, value).try_send() + } + + // count: + + pub fn count_with_tags<'a, T: ToCounterValue>( + &'a self, + key: &'a str, + value: T, + ) -> MetricBuilder<'a, 'a, Counter> { + self.base_tags + .iter() + .fold(self.wrapped.count_with_tags(key, value), |acc, (k, v)| { + acc.with_tag(k, v) + }) + } + + pub fn count<'a, T: ToCounterValue>(&'a self, key: &'a str, value: T) -> MetricResult { + self.count_with_tags(key, value).try_send() + } + + pub fn incr(&self, key: &str) -> MetricResult { + self.incr_with_tags(key).try_send() + } + + pub fn incr_with_tags<'a>(&'a self, key: &'a str) -> MetricBuilder<'a, 'a, Counter> { + self.count_with_tags(key, 1) + } + + pub fn decr(&self, key: &str) -> MetricResult { + self.decr_with_tags(key).try_send() + } + + pub fn decr_with_tags<'a>(&'a self, key: &'a str) -> MetricBuilder<'a, 'a, Counter> { + self.count_with_tags(key, -1) + } + + // histogram: + pub fn histogram(&self, key: &str, value: T) -> MetricResult { + self.histogram_with_tags(key, value).try_send() + } + + pub fn histogram_with_tags<'a, T: ToHistogramValue>( + &'a self, + key: &'a str, + value: T, + ) -> MetricBuilder<'a, 'a, Histogram> { + self.base_tags.iter().fold( + self.wrapped.histogram_with_tags(key, value), + |acc, (k, v)| acc.with_tag(k, v), + ) + } +} diff --git a/osprey_coordinator/src/metrics/recycling_socket_metrics_sink/mod.rs b/osprey_coordinator/src/metrics/recycling_socket_metrics_sink/mod.rs new file mode 100644 index 0000000..9c34807 --- /dev/null +++ b/osprey_coordinator/src/metrics/recycling_socket_metrics_sink/mod.rs @@ -0,0 +1,215 @@ +use std::env; +use std::io::Write; +use std::io::{self, BufWriter}; +use std::net::{SocketAddr, ToSocketAddrs, UdpSocket}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use cadence::{ErrorKind, MetricError, MetricResult, MetricSink}; + +const DEFAULT_BUFFER_SIZE: usize = 512; +const SOCKET_RECYCLE_INTERVAL: Duration = Duration::from_secs(15); + +fn get_buffer_size_from_environ() -> usize { + env::var("CADENCE_DEFAULT_BUFFER_SIZE") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(DEFAULT_BUFFER_SIZE) +} + +fn get_addr(addr: A) -> MetricResult { + match addr.to_socket_addrs()?.next() { + Some(addr) => Ok(addr), + None => Err(MetricError::from(( + ErrorKind::InvalidInput, + "No socket addresses yielded", + ))), + } +} + +#[derive(Debug)] +pub struct UdpMetricSink { + addr: SocketAddr, + socket: UdpSocket, + last_socket_created: Instant, +} + +impl UdpMetricSink { + pub fn from(to_addr: A) -> MetricResult + where + A: ToSocketAddrs, + { + let addr = get_addr(to_addr)?; + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.set_nonblocking(true)?; + + Ok(UdpMetricSink { + addr, + socket, + last_socket_created: Instant::now(), + }) + } +} + +impl Write for UdpMetricSink { + fn write(&mut self, buf: &[u8]) -> io::Result { + if self.last_socket_created.elapsed() > SOCKET_RECYCLE_INTERVAL { + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.set_nonblocking(true)?; + self.socket = socket; + self.last_socket_created = Instant::now(); + } + self.socket.send_to(buf, &self.addr) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[derive(Debug)] +pub(crate) struct UdpWriteAdapter { + sink: UdpMetricSink, +} + +impl UdpWriteAdapter { + pub(crate) fn new(sink: UdpMetricSink) -> UdpWriteAdapter { + Self { sink } + } +} + +impl Write for UdpWriteAdapter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.sink.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.sink.flush() + } +} + +/// This [`MetricSink`] implementation is buffered. It will hold on top data in its internal buffer until it has enough +/// to send out. This means that if you require timely delivery of metrics in low-throughput scenarios you will need to +/// periodically call [`BufferedUdpMetricSink::flush`] +#[derive(Clone, Debug)] +pub struct BufferedUdpMetricSink { + buffer: Arc>>, +} + +impl BufferedUdpMetricSink { + pub fn from(sink_addr: A) -> MetricResult + where + A: ToSocketAddrs, + { + Self::with_capacity(sink_addr, get_buffer_size_from_environ()) + } + + pub fn with_capacity(sink_addr: A, cap: usize) -> MetricResult + where + A: ToSocketAddrs, + { + let sink = UdpMetricSink::from(sink_addr)?; + Ok(BufferedUdpMetricSink { + buffer: Arc::new(Mutex::new(MultiLineWriter::new( + UdpWriteAdapter::new(sink), + cap, + ))), + }) + } +} + +impl MetricSink for BufferedUdpMetricSink { + fn emit(&self, metric: &str) -> io::Result { + let mut writer = self.buffer.lock().unwrap(); + writer.write(metric.as_bytes()) + } + + fn flush(&self) -> io::Result<()> { + let mut writer = self.buffer.lock().unwrap(); + writer.flush() + } +} + +#[derive(Debug, Default)] +struct WriterMetrics { + inner_write: u64, + buf_write: u64, + flushed: u64, +} + +#[derive(Debug)] +pub(crate) struct MultiLineWriter +where + T: Write, +{ + written: usize, + capacity: usize, + metrics: WriterMetrics, + inner: BufWriter, + line_ending: Vec, +} + +impl MultiLineWriter +where + T: Write, +{ + pub(crate) fn new(inner: T, cap: usize) -> MultiLineWriter { + Self::with_ending(inner, cap, "\n") + } + + pub(crate) fn with_ending(inner: T, cap: usize, end: &str) -> MultiLineWriter { + MultiLineWriter { + written: 0, + capacity: cap, + metrics: WriterMetrics::default(), + inner: BufWriter::with_capacity(cap, inner), + line_ending: Vec::from(end.as_bytes()), + } + } + + #[allow(dead_code)] + fn get_ref(&self) -> &T { + self.inner.get_ref() + } + + #[allow(dead_code)] + fn get_metrics(&self) -> &WriterMetrics { + &self.metrics + } +} + +impl Write for MultiLineWriter +where + T: Write, +{ + fn write(&mut self, buf: &[u8]) -> io::Result { + let left = self.capacity - self.written; + let required = buf.len() + self.line_ending.len(); + + if required > self.capacity { + self.metrics.inner_write += 1; + Ok(self.inner.get_mut().write(buf)?) + } else { + if left < required { + self.flush()?; + } + self.metrics.buf_write += 1; + let write1 = self.inner.write(buf)?; + self.written += write1; + + let write2 = self.inner.write(&self.line_ending)?; + self.written += write2; + + Ok(write1) + } + } + + fn flush(&mut self) -> io::Result<()> { + if self.written > 0 { + self.metrics.flushed += 1; + self.inner.flush()?; + self.written = 0; + } + Ok(()) + } +} diff --git a/osprey_coordinator/src/metrics/string_interner/mod.rs b/osprey_coordinator/src/metrics/string_interner/mod.rs new file mode 100644 index 0000000..5a25c3d --- /dev/null +++ b/osprey_coordinator/src/metrics/string_interner/mod.rs @@ -0,0 +1,271 @@ +use std::{ + borrow::Cow, + collections::{hash_map::Entry, HashMap, HashSet}, + hash::Hash, +}; + +use fxhash::{FxHashMap, FxHashSet}; +use parking_lot::RwLock; + +use super::DynamicTagValue; + +/// A string interner is responsible for taking a String, and turning it into a &'static str. It does so by leaking +/// the String using [`Box::leak`], and then caching that leaked string, so when the same string is provided, it'll +/// dispense an already interned leaked &'static str. There are two kinds of interners that can be used: +/// - [`Simple`]: Simply interns the provided string as is. +/// - [`Transformed`]: transforms the string and caches the transformed result. +pub struct StringInterner { + inner: RwLock, + max_capacity: usize, +} + +/// A [`InternerImpl`] which memoizes the transformation to a &'static str. This is useful in places where we might +/// take a string and transform it for use as a metrics tag. The transformation function should be deterministic and +/// return the same outputs for the same inputs always. +pub struct Transformed { + interned: FxHashMap, + transformer: Box String + Send + Sync>, +} + +/// A [`InternerImpl`] which simply interns the string provided as is. +pub struct Simple { + interned: FxHashSet<&'static str>, +} + +/// The result of [`StringInterner::intern`]. +#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] +pub enum InternResult { + /// The string interning was successful. This makes no distinguishment for if the string was already interned + /// prior to this call, or if it was just interned now. + Interned(&'static str), + /// The interner cache is at capacity, and can no longer intern any new strings. + AtCapacity, +} + +impl DynamicTagValue for InternResult { + fn as_static_str(&self) -> &'static str { + match self { + InternResult::Interned(s) => *s, + InternResult::AtCapacity => "string_interner_at_capacity", + } + } +} + +/// The trait that interners must implement. +pub trait InternerImpl { + /// Try to get the interned string from the cache, returning None if the cache did not have the interned value. + fn try_intern(&self, string: &str) -> Option<&'static str>; + /// Interns the string and inserts it into the cache. If the string is already interned, + /// it will not modify the cache. + fn intern(&mut self, string: String) -> &'static str; + /// How many strings is this interner currently interning. + fn len(&self) -> usize; + /// Is this interner currently interning zero strings? + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl InternerImpl for Transformed { + fn len(&self) -> usize { + self.interned.len() + } + + fn try_intern(&self, string: &str) -> Option<&'static str> { + self.interned.get(string).copied() + } + + fn intern(&mut self, string: String) -> &'static str { + match self.interned.entry(string) { + Entry::Occupied(e) => e.get(), + Entry::Vacant(v) => { + let transformed = Box::leak((self.transformer)(v.key()).into_boxed_str()); + v.insert(transformed) + } + } + } +} + +impl InternerImpl for Simple { + fn len(&self) -> usize { + self.interned.len() + } + + fn try_intern(&self, string: &str) -> Option<&'static str> { + self.interned.get(string).copied() + } + + fn intern(&mut self, string: String) -> &'static str { + if let Some(s) = self.try_intern(&string) { + return s; + } + let leaked = Box::leak(string.into_boxed_str()); + self.interned.insert(leaked); + leaked + } +} + +impl StringInterner { + /// Creates a new [`StringInterner`] that uses the [`Transformed`] interner implementation. For more details, see + /// the docs for [`Transformed`]. + /// + /// # Example: + /// + /// ```rust + /// # use crate::metrics::{StringInterner, InternResult}; + /// let interner = StringInterner::with_transformer(512, |s| s.to_uppercase()); + /// assert_eq!(interner.intern("hello"), InternResult::Interned("HELLO")); + /// ``` + pub fn with_transformer(max_capacity: usize, transformer: F) -> Self + where + F: Fn(&str) -> String + Send + Sync + 'static, + { + Self { + max_capacity, + inner: RwLock::new(Transformed { + interned: HashMap::with_capacity_and_hasher(max_capacity, Default::default()), + transformer: Box::new(transformer), + }), + } + } +} + +impl StringInterner { + /// Creates a new [`StringInterner`] that uses the [`Simple`] interner implementation. For more details, see + /// the docs for [`Simple`]. + /// + /// # Example: + /// + /// ```rust + /// # use crate::metrics::{StringInterner, InternResult}; + /// let interner = StringInterner::new(1); + /// assert_eq!(interner.intern("hello"), InternResult::Interned("hello")); + /// assert_eq!(interner.intern("world"), InternResult::AtCapacity); + /// ``` + pub fn new(max_capacity: usize) -> Self { + Self { + max_capacity, + inner: RwLock::new(Simple { + interned: HashSet::with_capacity_and_hasher(max_capacity, Default::default()), + }), + } + } +} + +impl StringInterner +where + T: InternerImpl, +{ + /// Interns the given string. + /// + /// This function takes a String or &str (basically something that can be turned into a Cow). If you can + /// provide an owned string, this is usually the best, as we won't have to clone the string when we're interning it, + /// however, if you can't, a borrowed string is fine too, and you'll only pay the cost of cloning the string the + /// first time its interned. + pub fn intern<'a>(&self, string: impl Into>) -> InternResult { + let string: Cow<'a, str> = string.into(); + + { + let read_only_interner = self.inner.read(); + if let Some(interned) = read_only_interner.try_intern(&string) { + return InternResult::Interned(interned); + } + + // If the interner is at capacity, bail early. + if read_only_interner.len() >= self.max_capacity { + return InternResult::AtCapacity; + } + } + + let mut interner = self.inner.write(); + // Check the length again, if we're full, bail before we try interning. + if interner.len() >= self.max_capacity { + return InternResult::AtCapacity; + } + + InternResult::Interned(interner.intern(string.into_owned())) + } +} + +/// Cache of strings for types that have a [`ToString`] implementation. +#[derive(Debug)] +pub struct ToStringInterner { + inner: RwLock>, + max_capacity: usize, +} + +impl ToStringInterner { + /// Create a new interner with the given capacity. + pub fn new(max_capacity: usize) -> Self { + ToStringInterner { + inner: RwLock::new(FxHashMap::with_capacity_and_hasher( + max_capacity, + Default::default(), + )), + max_capacity, + } + } + + /// Returns a static string for the given integer. + pub fn intern(&self, item: T) -> InternResult { + // Fast path: number already interned. + { + let map = self.inner.read(); + if let Some(s) = map.get(&item).copied() { + return InternResult::Interned(s); + } + + // If the interner is at capacity, bail and don't acquire write lock. + if map.len() >= self.max_capacity { + return InternResult::AtCapacity; + } + } + + let stringified_item = item.to_string(); // Perform string conversion outside critical section. + let mut map = self.inner.write(); + if map.len() >= self.max_capacity { + return InternResult::AtCapacity; + } + InternResult::Interned( + map.entry(item) + .or_insert_with(|| Box::leak(stringified_item.into_boxed_str())), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_to_string_interner_simple_case() { + let interner: ToStringInterner = ToStringInterner::new(2); + + // Inserts a new value + assert_eq!(interner.intern(1).as_static_str(), "1"); + assert_eq!(interner.inner.read().len(), 1); + + // Reads the previously-inserted value + assert_eq!(interner.intern(1).as_static_str(), "1"); + assert_eq!(interner.inner.read().len(), 1); + + // Inserts a new value + assert_eq!(interner.intern(2).as_static_str(), "2"); + assert_eq!(interner.inner.read().len(), 2); + + // The original value is still correct + assert_eq!(interner.intern(1).as_static_str(), "1"); + } + + #[test] + fn test_to_string_interner_over_capacity() { + let interner: ToStringInterner = ToStringInterner::new(2); + + assert_eq!(interner.intern(1).as_static_str(), "1"); + assert_eq!(interner.intern(2).as_static_str(), "2"); + assert_eq!( + interner.intern(3).as_static_str(), + "string_interner_at_capacity" + ); + } +} diff --git a/osprey_coordinator/src/metrics/v2/counter.rs b/osprey_coordinator/src/metrics/v2/counter.rs new file mode 100644 index 0000000..fb690d5 --- /dev/null +++ b/osprey_coordinator/src/metrics/v2/counter.rs @@ -0,0 +1,227 @@ +use std::{ + collections::HashMap, + sync::atomic::{AtomicI64, Ordering}, +}; + +use cadence::Counted; +use fxhash::FxHashMap; +use parking_lot::RwLock; + +use crate::metrics::v2::{send_submetric, BaseMetric, TagKey}; + +/// An integer counter. +#[derive(Debug)] +pub struct Counter { + storage: RwLock>, +} + +impl Counter { + #[inline] + pub fn new() -> Self { + Counter { + storage: Default::default(), + } + } + + /// Increments the counter by one. + /// + /// # Example + /// + /// ``` + /// # use crate::metrics::v2::Counter; + /// let counter = Counter::<()>::new(); + /// counter.incr(()); + /// assert_eq!(counter.snapshot().get(&()).copied(), Some(1)); + /// ``` + #[inline] + pub fn incr(&self, key: K) { + self.incr_by(key, 1); + } + + /// Increments the counter by the given value. + pub fn incr_by(&self, key: K, value: i64) { + if value == 0 { + return; + } + + // Fast path: key we've already seen before. + { + let locked = self.storage.read(); + if let Some(counter) = locked.get(&key) { + counter.fetch_add(value, Ordering::Relaxed); + return; + } + } + + let mut locked = self.storage.write(); + locked + .entry(key) + .and_modify(|counter| { + counter.fetch_add(value, Ordering::Relaxed); + }) + .or_insert(AtomicI64::new(value)); + } + + // Decrements the counter by one. + #[inline] + pub fn decr(&self, key: K) { + self.incr_by(key, -1); + } + + // Decrements the counter by the given value. + #[inline] + pub fn decr_by(&self, key: K, value: i64) { + self.incr_by(key, -value); + } + + /// Reads the current values of the metrics. + /// This is primarily intended for testing + /// and may have eventually consistent results + /// unless explicit synchronization is used. + /// + /// # Example + /// + /// ``` + /// # use crate::metrics::v2::Counter; + /// let counter = Counter::<()>::new(); + /// counter.incr(()); + /// assert_eq!(counter.snapshot().get(&()).copied(), Some(1)); + /// ``` + pub fn snapshot(&self) -> HashMap + where + K: Clone, + { + let storage = self.storage.read(); + storage + .iter() + .map(|(k, v)| (k.clone(), v.load(Ordering::Relaxed))) + .collect() + } +} + +// Manual implementation because we do not require `K: Default`. +impl Default for Counter { + fn default() -> Self { + Counter::new() + } +} + +impl BaseMetric for Counter { + fn send_metric( + &self, + client: &cadence::StatsdClient, + name: &str, + base_tags: &mut dyn Iterator, + ) -> Result<(), cadence::MetricError> { + let storage = self.storage.read(); + let base_tags: Vec<_> = base_tags.collect(); + for (key, counter) in storage.iter() { + let count = counter.swap(0, Ordering::Relaxed); + if count == 0 { + continue; + } + + send_submetric( + client.count_with_tags(name, count), + base_tags.iter().copied(), + key, + ); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::iter; + + use cadence::{SpyMetricSink, StatsdClient}; + + use super::*; + use crate::metrics::v2::statsd_output::StatsdOutput; + + #[test] + fn counter_send_zero() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let counter = Counter::<()>::new(); + + counter + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!(got, StatsdOutput::new()); + } + + #[test] + fn counter_send_one() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let counter = Counter::<()>::new(); + + counter.incr(()); + counter + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!(got, StatsdOutput::from([(b"foo:1|c".to_vec(), 1),])); + } + + #[derive(Clone, Debug, Hash, PartialEq, Eq)] + struct TestKey(&'static str, &'static str); + + impl TagKey for TestKey { + type TagIterator<'a> = iter::Once<(&'a str, &'a str)>; + + fn tags<'a>(&'a self) -> Self::TagIterator<'a> { + iter::once((self.0, self.1)) + } + } + + #[test] + fn counter_send_tagged() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let counter = Counter::::new(); + + counter.incr(TestKey("mykey", "xyzzy")); + counter + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([(b"foo:1|c|#mykey:xyzzy".to_vec(), 1),]) + ); + } + + #[test] + fn counter_send_multiple_tagged_with_base() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let counter = Counter::::new(); + let base_tags = &[("base", "ball")]; + + counter.incr(TestKey("mykey", "xyzzy")); + counter.incr(TestKey("mykey", "another")); + counter + .send_metric(&client, "foo", &mut base_tags.iter().copied()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo:1|c|#base:ball,mykey:xyzzy".to_vec(), 1), + (b"foo:1|c|#base:ball,mykey:another".to_vec(), 1) + ]) + ); + } +} diff --git a/osprey_coordinator/src/metrics/v2/gauge.rs b/osprey_coordinator/src/metrics/v2/gauge.rs new file mode 100644 index 0000000..b5f484e --- /dev/null +++ b/osprey_coordinator/src/metrics/v2/gauge.rs @@ -0,0 +1,371 @@ +use std::{ + collections::HashMap, + ops::{Add, Sub}, +}; + +use cadence::Gauged; +use fxhash::FxHashMap; +use parking_lot::{Mutex, RwLock}; + +use crate::metrics::v2::{send_submetric, BaseMetric, TagKey}; + +use self::private::Sealed; + +/// The type of a gauge: either a `u64` or an `f64`. +/// This trait is sealed and not meant to be implemented by other crates. +pub trait GaugeValue: + cadence::ext::ToGaugeValue + Add + Sub + PartialEq + PartialOrd + Copy + Sealed +{ +} + +impl GaugeValue for u64 {} +impl GaugeValue for f64 {} + +// Grouped metrics for a gauge at a particular instant. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GaugeSnapshot { + current: V, + min: V, + max: V, +} + +impl + Sub> GaugeSnapshot { + fn new(v: V) -> Self { + GaugeSnapshot { + current: v, + min: v, + max: v, + } + } + + fn set(&mut self, v: V) { + self.current = v; + if v < self.min { + self.min = v; + } + if v > self.max { + self.max = v; + } + } + + fn incr(&mut self, v: V) { + self.set(self.current.add(v)); + } + + fn decr(&mut self, v: V) { + self.set(self.current.sub(v)) + } + + /// Returns the most recently set value of the gauge. + pub fn current(&self) -> V { + self.current + } + + /// Returns the low watermark for the gauge. + pub fn min(&self) -> V { + self.min + } + + /// Returns the high watermark for the gauge. + pub fn max(&self) -> V { + self.max + } +} + +/// A metric that is reported continuously, +/// like disk space or memory used. +/// For a base name of `foo`, this type will send three gauges to Datadog: +/// +/// - `foo`: the most recent value of `foo` +/// - `foo.min`: the lowest value of `foo` observed since the gauge's creation +/// - `foo.max`: the highest value of `foo` observed since the gauge's creation +/// +/// # Example +/// +/// ``` +/// # use crate::metrics::v2::Gauge; +/// let gauge = Gauge::<(), f64>::new(); +/// gauge.set((), 3.142); +/// gauge.set((), 2.718); +/// ``` +#[derive(Debug)] +pub struct Gauge { + storage: RwLock>>>, +} + +impl + Sub> Gauge { + #[inline] + pub fn new() -> Self { + Gauge { + storage: Default::default(), + } + } + + pub fn set(&self, key: K, value: V) { + // Fast path: key we've already seen before. + { + let locked = self.storage.read(); + if let Some(snapshot) = locked.get(&key) { + snapshot.lock().set(value); + return; + } + } + + let mut locked = self.storage.write(); + locked + .entry(key) + .and_modify(|snapshot| { + snapshot.lock().set(value); + }) + .or_insert(Mutex::new(GaugeSnapshot::new(value))); + } + + pub fn incr(&self, key: K, value: V) { + { + let locked = self.storage.read(); + if let Some(snapshot) = locked.get(&key) { + snapshot.lock().incr(value); + return; + } + } + + let mut locked = self.storage.write(); + locked + .entry(key) + .and_modify(|snapshot| { + snapshot.lock().incr(value); + }) + .or_insert(Mutex::new(GaugeSnapshot::new(value))); + } + + pub fn decr(&self, key: K, value: V) { + { + let locked = self.storage.read(); + if let Some(snapshot) = locked.get(&key) { + snapshot.lock().decr(value); + return; + } + } + + let mut locked = self.storage.write(); + locked + .entry(key) + .and_modify(|snapshot| { + snapshot.lock().decr(value); + }) + .or_insert(Mutex::new(GaugeSnapshot::new(value))); + } + + /// Reads the current values of the metrics. + /// This is primarily intended for testing. + /// + /// # Example + /// + /// ``` + /// # use crate::metrics::v2::Gauge; + /// let gauge = Gauge::<(), u64>::new(); + /// gauge.set((), 42); + /// assert_eq!(gauge.snapshot().get(&()).map(|x| x.current()), Some(42)); + /// ``` + pub fn snapshot(&self) -> HashMap> + where + K: Clone, + { + let storage = self.storage.read(); + storage + .iter() + .map(|(k, v)| (k.clone(), v.lock().clone())) + .collect() + } +} + +// Manual implementation because we do not require `K: Default` nor `V: Default`. +impl + Sub> Default for Gauge { + fn default() -> Self { + Gauge::new() + } +} + +impl + Sub> BaseMetric for Gauge { + fn send_metric( + &self, + client: &cadence::StatsdClient, + name: &str, + base_tags: &mut dyn Iterator, + ) -> Result<(), cadence::MetricError> { + let storage = self.storage.read(); + let base_tags: Vec<_> = base_tags.collect(); + for (key, snapshot) in storage.iter() { + let snapshot = snapshot.lock().clone(); + send_submetric( + client.gauge_with_tags(name, snapshot.current()), + base_tags.iter().copied(), + key, + ); + send_submetric( + client.gauge_with_tags(&format!("{name}.min"), snapshot.min()), + base_tags.iter().copied(), + key, + ); + send_submetric( + client.gauge_with_tags(&format!("{name}.max"), snapshot.max()), + base_tags.iter().copied(), + key, + ); + } + Ok(()) + } +} + +mod private { + pub trait Sealed {} + + impl Sealed for u64 {} + + impl Sealed for f64 {} +} + +#[cfg(test)] +mod tests { + use std::iter; + + use cadence::{SpyMetricSink, StatsdClient}; + + use super::*; + use crate::metrics::v2::statsd_output::StatsdOutput; + + #[test] + fn gauge_snapshot_set_many() { + let mut snapshot = GaugeSnapshot::new(42u64); + snapshot.set(1); + snapshot.set(100); + snapshot.set(40); + assert_eq!(snapshot.current(), 40); + assert_eq!(snapshot.min(), 1); + assert_eq!(snapshot.max(), 100); + } + + #[test] + fn gauge_send_nothing() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let gauge = Gauge::<(), u64>::new(); + + gauge + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!(got, StatsdOutput::new()); + } + + #[test] + fn gauge_set_once() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let gauge = Gauge::<(), u64>::new(); + + gauge.set((), 42); + gauge + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo:42|g".to_vec(), 1), + (b"foo.min:42|g".to_vec(), 1), + (b"foo.max:42|g".to_vec(), 1), + ]) + ); + } + + #[test] + fn gauge_set_twice() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let gauge = Gauge::<(), u64>::new(); + + gauge.set((), 42); + gauge.set((), 100); + gauge + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo:100|g".to_vec(), 1), + (b"foo.min:42|g".to_vec(), 1), + (b"foo.max:100|g".to_vec(), 1), + ]) + ); + } + + #[derive(Clone, Debug, Hash, PartialEq, Eq)] + struct TestKey(&'static str, &'static str); + + impl TagKey for TestKey { + type TagIterator<'a> = iter::Once<(&'a str, &'a str)>; + + fn tags<'a>(&'a self) -> Self::TagIterator<'a> { + iter::once((self.0, self.1)) + } + } + + #[test] + fn gauge_send_tagged() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let gauge = Gauge::::new(); + + gauge.set(TestKey("mykey", "xyzzy"), 42); + gauge + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo:42|g|#mykey:xyzzy".to_vec(), 1), + (b"foo.min:42|g|#mykey:xyzzy".to_vec(), 1), + (b"foo.max:42|g|#mykey:xyzzy".to_vec(), 1), + ]) + ); + } + + #[test] + fn gauge_send_multiple_tagged_with_base() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let gauge = Gauge::::new(); + let base_tags = &[("base", "ball")]; + + gauge.set(TestKey("mykey", "xyzzy"), 42); + gauge.set(TestKey("mykey", "another"), 100); + gauge + .send_metric(&client, "foo", &mut base_tags.iter().copied()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo:42|g|#base:ball,mykey:xyzzy".to_vec(), 1), + (b"foo.min:42|g|#base:ball,mykey:xyzzy".to_vec(), 1), + (b"foo.max:42|g|#base:ball,mykey:xyzzy".to_vec(), 1), + (b"foo:100|g|#base:ball,mykey:another".to_vec(), 1), + (b"foo.min:100|g|#base:ball,mykey:another".to_vec(), 1), + (b"foo.max:100|g|#base:ball,mykey:another".to_vec(), 1) + ]) + ); + } +} diff --git a/osprey_coordinator/src/metrics/v2/histogram/duration.rs b/osprey_coordinator/src/metrics/v2/histogram/duration.rs new file mode 100644 index 0000000..6171026 --- /dev/null +++ b/osprey_coordinator/src/metrics/v2/histogram/duration.rs @@ -0,0 +1,179 @@ +use std::collections::HashMap; +use std::convert::TryFrom; +use std::num::NonZeroU64; +use std::time::Duration; + +use tracing::warn; + +use crate::metrics::v2::{BaseMetric, TagKey}; + +use super::{Histogram, HistogramBuilder, HistogramSnapshot}; + +/// A [`Histogram`] preconfigured for handling timing measurements. +/// +/// # Example +/// +/// ``` +/// # use std::time::Duration; +/// # use crate::metrics::v2::{DurationHistogram, DurationResolution}; +/// let histogram = DurationHistogram::<()>::new(DurationResolution::Milliseconds); +/// histogram.record((), Duration::from_millis(3142)); +/// histogram.record((), Duration::from_secs(27)); +/// ``` +#[derive(Debug)] +pub struct DurationHistogram { + resolution: DurationResolution, + inner: Histogram, +} + +impl DurationHistogram { + /// Returns a new histogram with the given resolution. + pub fn new(resolution: DurationResolution) -> Self { + let high = match resolution { + // [1 msec..1 hour] + DurationResolution::Milliseconds => 60 * 60 * 1_000, + // [1 usec..10 minutes] + DurationResolution::Microseconds => 10 * 60 * 1_000_000, + }; + DurationHistogram { + resolution, + inner: HistogramBuilder::new() + .range( + NonZeroU64::new(1).expect("constant"), + NonZeroU64::new(high).expect("constant"), + ) + .sigfig(2) + .create(), + } + } + + /// Records a value in the histogram. + pub fn record(&self, key: K, duration: Duration) { + let v128 = match self.resolution { + DurationResolution::Milliseconds => duration.as_millis(), + DurationResolution::Microseconds => duration.as_micros(), + }; + match u64::try_from(v128) { + Ok(v) => { + self.inner.record(key, v); + } + Err(_) => { + warn!( + "Unable to store duration {:?} in histogram at {:?} resolution", + duration, self.resolution + ); + } + } + } + + /// Reads the current values of the metrics. + /// This is primarily intended for testing. + /// The integers will be in units of the histogram's resolution. + #[inline] + pub fn snapshot(&self) -> HashMap + where + K: Clone, + { + self.inner.snapshot() + } +} + +// Manual implementation because we do not require `K: Default`. +impl Default for DurationHistogram { + /// Returns a histogram with `Milliseconds` resolution. + fn default() -> Self { + DurationHistogram::new(DurationResolution::default()) + } +} + +impl BaseMetric for DurationHistogram { + fn send_metric( + &self, + client: &cadence::StatsdClient, + name: &str, + base_tags: &mut dyn Iterator, + ) -> Result<(), cadence::MetricError> { + self.inner.send_metric(client, name, base_tags) + } +} + +/// The resolution of a [`DurationHistogram`]. +/// Lower values are less precise. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(u8)] +#[non_exhaustive] +pub enum DurationResolution { + Milliseconds, + Microseconds, +} + +impl Default for DurationResolution { + /// Returns `Milliseconds`. + fn default() -> Self { + DurationResolution::Milliseconds + } +} + +#[cfg(test)] +mod tests { + use std::iter; + + use cadence::{SpyMetricSink, StatsdClient}; + + use super::*; + use crate::metrics::v2::statsd_output::StatsdOutput; + + #[test] + fn test_milliseconds() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let histogram = DurationHistogram::<()>::new(DurationResolution::Milliseconds); + + histogram.record((), Duration::from_millis(42)); + histogram + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo.count:1|c".to_vec(), 1), + (b"foo.min:42|g".to_vec(), 1), + (b"foo.max:42|g".to_vec(), 1), + (b"foo.avg:42|g".to_vec(), 1), + (b"foo.median:42|g".to_vec(), 1), + (b"foo.95percentile:42|g".to_vec(), 1), + (b"foo.99percentile:42|g".to_vec(), 1), + ]) + ); + } + + #[test] + fn test_microseconds() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let histogram = DurationHistogram::<()>::new(DurationResolution::Microseconds); + + histogram.record((), Duration::from_micros(42)); + histogram + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo.count:1|c".to_vec(), 1), + (b"foo.min:42|g".to_vec(), 1), + (b"foo.max:42|g".to_vec(), 1), + (b"foo.avg:42|g".to_vec(), 1), + (b"foo.median:42|g".to_vec(), 1), + (b"foo.95percentile:42|g".to_vec(), 1), + (b"foo.99percentile:42|g".to_vec(), 1), + ]) + ); + } +} diff --git a/osprey_coordinator/src/metrics/v2/histogram/mod.rs b/osprey_coordinator/src/metrics/v2/histogram/mod.rs new file mode 100644 index 0000000..c319b26 --- /dev/null +++ b/osprey_coordinator/src/metrics/v2/histogram/mod.rs @@ -0,0 +1,534 @@ +mod duration; + +use std::num::NonZeroU64; +use std::{collections::HashMap, convert::TryFrom}; + +use cadence::{Counted, Gauged}; +use fxhash::FxHashMap; +use hdrhistogram::{Counter, Histogram as HdrHistogram}; +use parking_lot::{Mutex, RwLock}; +use tracing::warn; + +use crate::metrics::v2::{send_submetric, BaseMetric, TagKey}; + +pub use self::duration::*; + +// Grouped metrics for a histogram at a particular instant. +#[derive(Clone, Debug)] +pub struct HistogramSnapshot { + sample_count: NonZeroU64, + min: u64, + max: u64, + mean: f64, + median: u64, + p95: u64, + p99: u64, +} + +impl HistogramSnapshot { + fn from_histogram(h: &HdrHistogram) -> Option { + if let Some(sample_count) = NonZeroU64::new(h.len()) { + Some(HistogramSnapshot { + sample_count, + min: h.min(), + max: h.max(), + mean: h.mean(), + median: h.value_at_quantile(0.5), + p95: h.value_at_quantile(0.95), + p99: h.value_at_quantile(0.99), + }) + } else { + None + } + } + + /// Returns the number of values recorded + /// since the last send of the histogram. + #[inline] + pub fn sample_count(&self) -> NonZeroU64 { + self.sample_count + } + + /// Returns the minimum + /// of values recorded since the last send of the histogram. + #[inline] + pub fn min(&self) -> u64 { + self.min + } + + /// Returns the maximum + /// of values recorded since the last send of the histogram. + #[inline] + pub fn max(&self) -> u64 { + self.max + } + + /// Returns the arithmetic mean (average) + /// of values recorded since the last send of the histogram. + #[inline] + pub fn mean(&self) -> f64 { + self.mean + } + + /// Returns the median (50th percentile) + /// of values recorded since the last send of the histogram. + #[inline] + pub fn median(&self) -> u64 { + self.median + } + + /// Returns the 95th percentile + /// of values recorded since the last send of the histogram. + #[inline] + pub fn p95(&self) -> u64 { + self.p95 + } + + /// Returns the 99th percentile + /// of values recorded since the last send of the histogram. + #[inline] + pub fn p99(&self) -> u64 { + self.p99 + } +} + +/// A metric that reports on a distribution of `u64` values. +/// For a base name of `foo`, this type will send seven metrics to Datadog: +/// +/// - `foo.count`: `COUNT` of the number of samples +/// - `foo.min`: `GAUGE` of the lowest value observed since the last send. +/// - `foo.max`: `GAUGE` of the highest value observed since the last send. +/// - `foo.avg`: `GAUGE` of the average value observed since the last send. +/// - `foo.median`: `GAUGE` of the median value observed since the last send. +/// - `foo.95percentile`: `GAUGE` of the P95 value observed since the last send. +/// - `foo.99percentile`: `GAUGE` of the P99 value observed since the last send. +/// +/// # Example +/// +/// ``` +/// # use crate::metrics::v2::Histogram; +/// let histogram = Histogram::<()>::new(); +/// histogram.record((), 3142); +/// histogram.record((), 2718); +/// ``` +#[derive(Debug)] +pub struct Histogram { + options: HistogramBuilder, + storage: RwLock>>>, +} + +impl Histogram { + /// Returns a new histogram with the default settings. + /// If you want to customize the precision, use [`HistogramBuilder`]. + pub fn new() -> Self { + HistogramBuilder::new().create() + } + + /// Records a value in the histogram. + pub fn record(&self, key: K, value: u64) { + // Fast path: key we've already seen before. + { + let locked = self.storage.read(); + if let Some(histogram_mutex) = locked.get(&key) { + if let Err(err) = histogram_mutex.lock().record(value) { + warn!("Failed to record histogram value: {}", err); + } + return; + } + } + + // Creating the histogram outside the critical section + // because we allocate (and can panic if we have a bug) + // without affecting others. + let mut new_histogram = HdrHistogram::new_with_bounds( + self.options.low.into(), + self.options.high.into(), + self.options.sigfig, + ) + .expect("builder should have validated parameters"); + if let Err(err) = new_histogram.record(value) { + warn!("Failed to record histogram value: {}", err); + return; + } + + let mut locked = self.storage.write(); + locked + .entry(key) + .and_modify(|histogram_mutex| { + if let Err(err) = histogram_mutex.lock().record(value) { + warn!("Failed to record histogram value: {}", err); + } + }) + .or_insert(Mutex::new(new_histogram)); + } + + /// Reads the current values of the metrics. + /// This is primarily intended for testing. + /// + /// # Example + /// + /// ``` + /// # use crate::metrics::v2::Histogram; + /// let histogram = Histogram::<()>::new(); + /// histogram.record((), 42); + /// assert_eq!(histogram.snapshot().get(&()).map(|x| x.p95()), Some(42)); + /// ``` + pub fn snapshot(&self) -> HashMap + where + K: Clone, + { + let storage = self.storage.read(); + storage + .iter() + .filter_map(|(k, mu)| { + HistogramSnapshot::from_histogram(&mu.lock()).map(|v| (k.clone(), v)) + }) + .collect() + } +} + +// Manual implementation because we do not require `K: Default`. +impl Default for Histogram { + fn default() -> Self { + Histogram::new() + } +} + +impl BaseMetric for Histogram { + fn send_metric( + &self, + client: &cadence::StatsdClient, + name: &str, + base_tags: &mut dyn Iterator, + ) -> Result<(), cadence::MetricError> { + let storage = self.storage.read(); + let base_tags: Vec<_> = base_tags.collect(); + for (key, histogram_mutex) in storage.iter() { + let snapshot = { + let mut histogram = histogram_mutex.lock(); + let snapshot = HistogramSnapshot::from_histogram(&histogram); + histogram.reset(); + snapshot + }; + let snapshot = match snapshot { + Some(snapshot) => snapshot, + None => continue, + }; + let sample_count = + i64::try_from(u64::from(snapshot.sample_count())).unwrap_or(i64::max_value()); + send_submetric( + client.count_with_tags(&format!("{}.count", name), sample_count), + base_tags.iter().copied(), + key, + ); + send_submetric( + client.gauge_with_tags(&format!("{}.avg", name), snapshot.mean()), + base_tags.iter().copied(), + key, + ); + send_submetric( + client.gauge_with_tags(&format!("{}.median", name), snapshot.median()), + base_tags.iter().copied(), + key, + ); + send_submetric( + client.gauge_with_tags(&format!("{}.95percentile", name), snapshot.p95()), + base_tags.iter().copied(), + key, + ); + send_submetric( + client.gauge_with_tags(&format!("{}.99percentile", name), snapshot.p99()), + base_tags.iter().copied(), + key, + ); + send_submetric( + client.gauge_with_tags(&format!("{}.min", name), snapshot.min()), + base_tags.iter().copied(), + key, + ); + send_submetric( + client.gauge_with_tags(&format!("{}.max", name), snapshot.max()), + base_tags.iter().copied(), + key, + ); + } + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub struct HistogramBuilder { + low: NonZeroU64, + high: NonZeroU64, + sigfig: u8, +} + +impl HistogramBuilder { + /// Creates a builder with the default options. + #[inline] + pub fn new() -> Self { + // Values recommended in + // https://docs.rs/hdrhistogram/7.5.2/hdrhistogram/struct.Histogram.html#method.new_with_bounds + HistogramBuilder { + low: NonZeroU64::new(1).expect("constant"), + high: NonZeroU64::new(u64::max_value()).expect("constant"), + sigfig: 2, + } + } + + /// Sets the range of the histogram's values. + /// `low` is the lowest value that can be distinguished from zero. + /// `high` is the highest value to be tracked and must be `>= 2 * low`. + /// + /// # Panics + /// + /// If `high` is not sufficiently larger than `low`. + pub fn range(&mut self, low: NonZeroU64, high: NonZeroU64) -> &mut Self { + assert!( + low.checked_mul(NonZeroU64::new(2).expect("constant")) + .map(|low2| high >= low2) + .unwrap_or(false), + "Maximum value of histogram must be at least 2x the minimum." + ); + self.low = low; + self.high = high; + self + } + + /// Sets the number of significant digits (after the decimal place) + /// to which the histogram will maintain value resolution and separation. + /// This will increase memory usage exponentially, so be careful. + /// + /// # Panics + /// + /// If `sigfig` is greater than 5. + pub fn sigfig(&mut self, sigfig: u8) -> &mut Self { + assert!(sigfig <= 5, "Histogram precision too large"); + self.sigfig = sigfig; + self + } + + /// Returns a new histogram with the builder's parameters. + pub fn create(&self) -> Histogram { + Histogram { + options: self.clone(), + storage: Default::default(), + } + } +} + +impl Default for HistogramBuilder { + fn default() -> Self { + HistogramBuilder::new() + } +} + +#[cfg(test)] +mod tests { + use std::iter; + + use cadence::{SpyMetricSink, StatsdClient}; + + use super::*; + use crate::metrics::v2::statsd_output::StatsdOutput; + + #[test] + fn histogram_send_nothing() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let histogram = Histogram::<()>::new(); + + histogram + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!(got, StatsdOutput::new()); + } + + #[test] + fn histogram_set_once() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let histogram = Histogram::<()>::new(); + + histogram.record((), 42); + histogram + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo.count:1|c".to_vec(), 1), + (b"foo.min:42|g".to_vec(), 1), + (b"foo.max:42|g".to_vec(), 1), + (b"foo.avg:42|g".to_vec(), 1), + (b"foo.median:42|g".to_vec(), 1), + (b"foo.95percentile:42|g".to_vec(), 1), + (b"foo.99percentile:42|g".to_vec(), 1), + ]) + ); + } + + #[test] + fn histogram_clears_on_send() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let histogram = Histogram::<()>::new(); + + histogram.record((), 42); + histogram + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo.count:1|c".to_vec(), 1), + (b"foo.min:42|g".to_vec(), 1), + (b"foo.max:42|g".to_vec(), 1), + (b"foo.avg:42|g".to_vec(), 1), + (b"foo.median:42|g".to_vec(), 1), + (b"foo.95percentile:42|g".to_vec(), 1), + (b"foo.99percentile:42|g".to_vec(), 1), + ]) + ); + + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + + histogram.record((), 10); + histogram + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo.count:1|c".to_vec(), 1), + (b"foo.min:10|g".to_vec(), 1), + (b"foo.max:10|g".to_vec(), 1), + (b"foo.avg:10|g".to_vec(), 1), + (b"foo.median:10|g".to_vec(), 1), + (b"foo.95percentile:10|g".to_vec(), 1), + (b"foo.99percentile:10|g".to_vec(), 1), + ]) + ); + } + + #[test] + fn histogram_set_twice() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let histogram = Histogram::<()>::new(); + + histogram.record((), 42); + histogram.record((), 100); + histogram + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo.count:2|c".to_vec(), 1), + (b"foo.min:42|g".to_vec(), 1), + (b"foo.max:100|g".to_vec(), 1), + (b"foo.median:42|g".to_vec(), 1), + (b"foo.avg:71|g".to_vec(), 1), + (b"foo.95percentile:100|g".to_vec(), 1), + (b"foo.99percentile:100|g".to_vec(), 1), + ]) + ); + } + + #[derive(Clone, Debug, Hash, PartialEq, Eq)] + struct TestKey(&'static str, &'static str); + + impl TagKey for TestKey { + type TagIterator<'a> = iter::Once<(&'a str, &'a str)>; + + fn tags<'a>(&'a self) -> Self::TagIterator<'a> { + iter::once((self.0, self.1)) + } + } + + #[test] + fn histogram_send_tagged() { + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let histogram = Histogram::::new(); + + histogram.record(TestKey("mykey", "xyzzy"), 42); + histogram + .send_metric(&client, "foo", &mut iter::empty()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo.count:1|c|#mykey:xyzzy".to_vec(), 1), + (b"foo.min:42|g|#mykey:xyzzy".to_vec(), 1), + (b"foo.max:42|g|#mykey:xyzzy".to_vec(), 1), + (b"foo.avg:42|g|#mykey:xyzzy".to_vec(), 1), + (b"foo.median:42|g|#mykey:xyzzy".to_vec(), 1), + (b"foo.95percentile:42|g|#mykey:xyzzy".to_vec(), 1), + (b"foo.99percentile:42|g|#mykey:xyzzy".to_vec(), 1), + ]) + ); + } + + #[test] + fn histogram_send_multiple_tagged_with_base() { + let (receiver, sink) = SpyMetricSink::with_capacity(100); + let client = StatsdClient::from_sink("", sink); + let histogram = Histogram::::new(); + let base_tags = &[("base", "ball")]; + + histogram.record(TestKey("mykey", "xyzzy"), 42); + histogram.record(TestKey("mykey", "another"), 100); + histogram + .send_metric(&client, "foo", &mut base_tags.iter().copied()) + .unwrap(); + drop(client); // close the sending end + + let got = StatsdOutput::from_packets(receiver); + assert_eq!( + got, + StatsdOutput::from([ + (b"foo.count:1|c|#base:ball,mykey:xyzzy".to_vec(), 1), + (b"foo.min:42|g|#base:ball,mykey:xyzzy".to_vec(), 1), + (b"foo.max:42|g|#base:ball,mykey:xyzzy".to_vec(), 1), + (b"foo.avg:42|g|#base:ball,mykey:xyzzy".to_vec(), 1), + (b"foo.median:42|g|#base:ball,mykey:xyzzy".to_vec(), 1), + (b"foo.95percentile:42|g|#base:ball,mykey:xyzzy".to_vec(), 1), + (b"foo.99percentile:42|g|#base:ball,mykey:xyzzy".to_vec(), 1), + (b"foo.count:1|c|#base:ball,mykey:another".to_vec(), 1), + (b"foo.min:100|g|#base:ball,mykey:another".to_vec(), 1), + (b"foo.max:100|g|#base:ball,mykey:another".to_vec(), 1), + (b"foo.avg:100|g|#base:ball,mykey:another".to_vec(), 1), + (b"foo.median:100|g|#base:ball,mykey:another".to_vec(), 1), + ( + b"foo.95percentile:100|g|#base:ball,mykey:another".to_vec(), + 1 + ), + ( + b"foo.99percentile:100|g|#base:ball,mykey:another".to_vec(), + 1 + ), + ]) + ); + } +} diff --git a/osprey_coordinator/src/metrics/v2/mod.rs b/osprey_coordinator/src/metrics/v2/mod.rs new file mode 100644 index 0000000..cf9e973 --- /dev/null +++ b/osprey_coordinator/src/metrics/v2/mod.rs @@ -0,0 +1,240 @@ +mod counter; +mod gauge; +mod histogram; +mod send; +#[cfg(test)] +pub(crate) mod statsd_output; + +use std::hash::Hash; +use std::iter::{self, Empty}; +use std::ops::Deref; +use std::sync::Arc; + +use bytes_utils::Str; +use cadence::{Metric, MetricBuilder, MetricError, StatsdClient}; + +pub use self::counter::*; +pub use self::gauge::*; +pub use self::histogram::*; +pub use self::send::*; + +/// A type that emits its own metrics. +/// Usually a collection of other named metrics, +/// like a struct. +pub trait MetricsEmitter { + fn emit_metrics( + &self, + client: &StatsdClient, + base_tags: &mut dyn Iterator, + ) -> Result<(), MetricError>; +} + +/// Boxes of types that implement `MetricsEmitter` implement `MetricsEmitter` +/// by calling `emit_metrics` on the inner value. +impl MetricsEmitter for Box { + fn emit_metrics( + &self, + client: &StatsdClient, + base_tags: &mut dyn Iterator, + ) -> Result<(), MetricError> { + self.as_ref().emit_metrics(client, base_tags) + } +} + +/// `Arc`s of types that implement `MetricsEmitter` implement `MetricsEmitter` +/// by calling `emit_metrics` on the inner value. +impl MetricsEmitter for Arc { + fn emit_metrics( + &self, + client: &StatsdClient, + base_tags: &mut dyn Iterator, + ) -> Result<(), MetricError> { + self.as_ref().emit_metrics(client, base_tags) + } +} + +#[derive(Clone, Debug)] +#[doc(hidden)] +pub struct NopMetricsEmitter; + +impl MetricsEmitter for NopMetricsEmitter { + fn emit_metrics( + &self, + _client: &StatsdClient, + _base_tags: &mut dyn Iterator, + ) -> Result<(), MetricError> { + Ok(()) + } +} + +/// Wraps a [`MetricsEmitter`] or [`BaseMetric`] with base tags. +#[derive(Clone, Debug)] +pub struct TagWrapper { + tags: Vec<(Str, Str)>, + inner: T, +} + +impl TagWrapper { + /// Wraps the given [`MetricsEmitter`] or [`BaseMetric`] with base tags. + /// + /// # Example + /// + /// ``` + /// # use crate::metrics::v2::TagWrapper; + /// # let emitter = crate::metrics::v2::NopMetricsEmitter; + /// TagWrapper::new([("foo", "bar")], emitter) + /// # ; + /// ``` + pub fn new(tags: impl IntoIterator, inner: T) -> Arc + where + K: Into, + V: Into, + { + Arc::new(TagWrapper { + tags: tags + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(), + inner, + }) + } + + /// Returns the inner emitter or metric. + #[inline] + pub fn inner(&self) -> &T { + &self.inner + } +} + +impl MetricsEmitter for TagWrapper { + fn emit_metrics( + &self, + client: &StatsdClient, + base_tags: &mut dyn Iterator, + ) -> Result<(), MetricError> { + let mut base_tags = base_tags + .map(|t| t) // to unify the lifetimes + .chain(self.tags.iter().map(|(k, v)| (k.deref(), v.deref()))); + self.inner.emit_metrics(client, &mut base_tags) + } +} + +impl BaseMetric for TagWrapper { + fn send_metric( + &self, + client: &StatsdClient, + name: &str, + base_tags: &mut dyn Iterator, + ) -> Result<(), MetricError> { + let mut base_tags = base_tags + .map(|t| t) // to unify the lifetimes + .chain(self.tags.iter().map(|(k, v)| (k.deref(), v.deref()))); + self.inner.send_metric(client, name, &mut base_tags) + } +} + +/// A single metric (e.g. a counter or a histogram). +pub trait BaseMetric { + /// Send the data of the metric under the given name. + fn send_metric( + &self, + client: &StatsdClient, + name: &str, + base_tags: &mut dyn Iterator, + ) -> Result<(), MetricError>; +} + +/// Boxes of types that implement `BaseMetric` implement `BaseMetric` +/// by calling `send_metric` on the inner value. +impl BaseMetric for Box { + fn send_metric( + &self, + client: &StatsdClient, + name: &str, + base_tags: &mut dyn Iterator, + ) -> Result<(), MetricError> { + self.as_ref().send_metric(client, name, base_tags) + } +} + +/// `Arc`s of types that implement `BaseMetric` implement `BaseMetric` +/// by calling `send_metric` on the inner value. +impl BaseMetric for Arc { + fn send_metric( + &self, + client: &StatsdClient, + name: &str, + base_tags: &mut dyn Iterator, + ) -> Result<(), MetricError> { + self.as_ref().send_metric(client, name, base_tags) + } +} + +/// A collection of key/value metric tags to apply to a raw number. +pub trait TagKey: Hash + Eq { + type TagIterator<'a>: Iterator + where + Self: 'a; + + fn tags<'a>(&'a self) -> Self::TagIterator<'a>; +} + +/// Helper trait to convert tag values to static strings. +/// This is implemented for types that can be used as tag values. +pub trait TagValue { + fn as_tag_str(&self) -> &'static str; +} + +impl TagValue for &'static str { + fn as_tag_str(&self) -> &'static str { + self + } +} + +impl TagValue for crate::metrics::string_interner::InternResult { + fn as_tag_str(&self) -> &'static str { + crate::metrics::DynamicTagValue::as_static_str(self) + } +} + +impl TagValue for tonic::Code { + fn as_tag_str(&self) -> &'static str { + crate::metrics::DynamicTagValue::as_static_str(self) + } +} + +impl TagValue for bool { + fn as_tag_str(&self) -> &'static str { + crate::metrics::DynamicTagValue::as_static_str(self) + } +} + +/// The unit type can be used as an empty set of metric tags. +impl TagKey for () { + type TagIterator<'a> = Empty<(&'a str, &'a str)>; + + fn tags<'a>(&'a self) -> Self::TagIterator<'a> { + iter::empty() + } +} + +/// Helper function to collect a set of base tags and a key's tags together +/// and send the metric. +pub(crate) fn send_submetric<'m, T: Metric + From>( + mut builder: MetricBuilder<'m, '_, T>, + base_tags: impl IntoIterator, + key: &'m impl TagKey, +) { + for (base_key, base_value) in base_tags { + builder = builder.with_tag(base_key, base_value); + } + for (k, v) in key.tags() { + builder = builder.with_tag(k, v); + } + builder.try_send().ok(); +} + +#[doc(hidden)] +pub mod macro_support { + pub use cadence::{MetricError, StatsdClient}; +} diff --git a/osprey_coordinator/src/metrics/v2/send.rs b/osprey_coordinator/src/metrics/v2/send.rs new file mode 100644 index 0000000..b51f307 --- /dev/null +++ b/osprey_coordinator/src/metrics/v2/send.rs @@ -0,0 +1,291 @@ +use std::collections::HashMap; +use std::env; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; +use std::{borrow::Borrow, collections::hash_map::Entry}; + +use bytes_utils::Str; +use cadence::{MetricSink, StatsdClient}; +use lazy_static::lazy_static; +use parking_lot::RwLock; +use thiserror::Error; +use tracing::error; + +use crate::metrics::recycling_socket_metrics_sink::BufferedUdpMetricSink; +use crate::metrics::v2::MetricsEmitter; + +const METRIC_REPORTING_FREQUENCY: Duration = Duration::from_millis(300); + +lazy_static! { + static ref EMITTERS: RwLock>> = + RwLock::new(HashMap::new()); +} + +static SENDER_RUNNING: AtomicBool = AtomicBool::new(false); + +/// Registers a metric emitter to be sent by [`MetricsSender`]. +/// This is safe to call concurrently with a running [`MetricsSender`]. +/// +/// Panics if the same emitter is registered twice. +pub fn register_emitter( + emitter: Arc, +) -> RegisteredEmitter { + let id = unsafe { std::mem::transmute(Arc::as_ptr(&emitter)) }; + tracing::debug!("registered {id} as emitter"); + match EMITTERS.write().entry(id) { + Entry::Occupied(_) => panic!("This emitter has already been registered."), + Entry::Vacant(ent) => { + ent.insert(emitter); + } + } + RegisteredEmitter { id } +} + +/// A handle to an emitter registed with [`register_emitter`]. +/// Dropping the handle will cause the metrics to stop being emitted. +/// If this is undesirable, you can use [`RegisteredEmitter::emit_forever`] +/// to emit the metrics until the [`MetricsSender`] stops running. +#[derive(Debug)] +#[must_use] +pub struct RegisteredEmitter { + id: u128, +} + +impl RegisteredEmitter { + /// Keep emitting metrics and consume this handle. + pub fn emit_forever(mut self) { + self.id = 0; + } +} + +impl Drop for RegisteredEmitter { + fn drop(&mut self) { + if self.id != 0 { + tracing::debug!("dropping emitter: {}", self.id); + EMITTERS.write().remove(&self.id); + } + } +} + +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum SendMetricsError { + #[error("metrics are already being sent")] + AlreadySending, + #[error("{0}")] + Other(#[source] Box), +} + +/// Builder for [`MetricsSender`]. +#[derive(Debug, Default)] +pub struct MetricsSenderBuilder { + client: Option, + base_tags: Vec<(Str, Str)>, + exclude_standard_tags: bool, +} + +impl MetricsSenderBuilder { + /// Returns a new builder. + #[inline] + pub fn new() -> Self { + Default::default() + } + + /// Sets the statsd client to use. + /// The default is derived from the `DD_AGENT_HOST` environment variable. + pub fn client(mut self, client: StatsdClient) -> Self { + self.client = Some(client); + self + } + + /// Excludes the `dd.internal.entity_id` and `owner` tags + /// that are automatically derived from environment variables. + /// If you don't provide these tags, + /// Datadog will not set the `env`, `service`, or `version` tags. + pub fn without_standard_tags(mut self) -> Self { + self.exclude_standard_tags = true; + self + } + + /// Add a single tag to be attached to all sent metrics. + /// This method may be called multiple times to add multiple tags. + pub fn tag(mut self, k: impl Into, v: impl Into) -> Self { + self.base_tags.push((k.into(), v.into())); + self + } + + /// Add tags to be attached to all sent metrics. + /// This method may be called multiple times to add multiple sets of tags. + pub fn add_tags, V: Into>( + mut self, + tags: impl IntoIterator, + ) -> Self { + self.base_tags + .extend(tags.into_iter().map(|(k, v)| (k.into(), v.into()))); + self + } + + /// Spawn a thread to periodically send the metrics registered with [`register_emitter`]. + pub fn start(self) -> Result { + let MetricsSenderBuilder { + client, + mut base_tags, + exclude_standard_tags, + } = self; + if !exclude_standard_tags { + if let Ok(tag) = env::var("DD_ENTITY_ID") { + base_tags.insert( + 0, + (Str::from_static("dd.internal.entity_id"), Str::from(tag)), + ); + } + } + let (flush_sink, client) = if let Some(client) = client { + (None, client) + } else { + let host = std::env::var("DD_AGENT_HOST") + .or_else(|_| std::env::var("DATADOG_TRACE_AGENT_HOSTNAME")) + .unwrap_or_else(|_| "127.0.0.1".into()); + let host = (host.as_ref(), 8125); + tracing::debug!("connecting to statsd on {host:?}"); + let sink = BufferedUdpMetricSink::from(host) + .map_err(|e| SendMetricsError::Other(Box::new(e)))?; + (Some(sink.clone()), StatsdClient::from_sink("", sink)) + }; + + if SENDER_RUNNING + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + return Err(SendMetricsError::AlreadySending); + } + + let interrupt_tx = Arc::new(AtomicBool::new(false)); + let interrupt_rx = interrupt_tx.clone(); + let join_handle = thread::spawn(move || { + tracing::debug!("reporting metrics every {METRIC_REPORTING_FREQUENCY:#?}"); + let mut next_emit = Instant::now() + METRIC_REPORTING_FREQUENCY; + // Do a force flush of metrics in our buffers every 3 ticks to ensure we still report metrics in + // low-throughput scenarios. + let mut next_flush = Instant::now() + (METRIC_REPORTING_FREQUENCY * 3); + loop { + if let Some(remaining) = next_emit.checked_duration_since(Instant::now()) { + if interrupt_rx.load(Ordering::SeqCst) { + tracing::debug!("metrics-sender thread interrupted, exiting"); + return; + } + thread::park_timeout(remaining); + } + if interrupt_rx.load(Ordering::SeqCst) { + tracing::debug!("metrics-sender thread interrupted, exiting"); + return; + } + let now = Instant::now(); + if now < next_emit { + // Park can be awoken spuriously. + continue; + } + + next_emit = now + METRIC_REPORTING_FREQUENCY; + for emitter in EMITTERS.read().values() { + let result = emitter.emit_metrics( + &client, + &mut base_tags.iter().map(|(k, v)| (k.borrow(), v.borrow())), + ); + if let Err(err) = result { + error!("emitting metrics failed: {err}"); + } + if interrupt_rx.load(Ordering::SeqCst) { + tracing::debug!("metrics-sender thread interrupted, exiting"); + return; + } + } + // Now attempt to force-flush to ensure timely delivery of metrics in low-throughput scenarios + if now >= next_flush { + if let Some(sink) = flush_sink.as_ref() { + if let Err(e) = sink.flush() { + tracing::error!("failed to flush metrics: {e}") + } + } + next_flush = now + (METRIC_REPORTING_FREQUENCY * 3); + } + } + }); + + Ok(MetricsSender { + interrupt: interrupt_tx, + join_handle: Some(join_handle), + }) + } +} + +/// Handle to a running metrics sender thread. +/// `Drop`ping the handle will stop the thread. +#[derive(Debug)] +#[must_use = "Dropping a MetricsSender stops its thread"] +pub struct MetricsSender { + interrupt: Arc, + join_handle: Option>, +} + +impl MetricsSender { + /// Return a new sender builder. + #[inline] + pub fn builder() -> MetricsSenderBuilder { + MetricsSenderBuilder::new() + } +} + +impl Drop for MetricsSender { + fn drop(&mut self) { + tracing::debug!("MetricsSender dropped, shutting down"); + self.interrupt.store(true, Ordering::SeqCst); + let join_handle = self.join_handle.take().unwrap(); + join_handle.thread().unpark(); + join_handle.join().ok(); + SENDER_RUNNING.store(false, Ordering::SeqCst); + } +} + +#[cfg(test)] +mod tests { + use std::iter; + + use cadence::{Counted, SpyMetricSink, StatsdClient}; + + use super::*; + use crate::metrics::v2::statsd_output::StatsdOutput; + + #[test] + fn test_send() { + struct MyMetricsEmitter {} + + impl MetricsEmitter for MyMetricsEmitter { + fn emit_metrics( + &self, + client: &cadence::StatsdClient, + base_tags: &mut dyn Iterator, + ) -> Result<(), cadence::MetricError> { + let mut builder = client.count_with_tags("foo", 1); + for (k, v) in base_tags { + builder = builder.with_tag(k, v); + } + builder.try_send()?; + Ok(()) + } + } + + let emitter = Arc::new(MyMetricsEmitter {}); + let _registration = register_emitter(emitter); + + let (receiver, sink) = SpyMetricSink::with_capacity(10); + let client = StatsdClient::from_sink("", sink); + let _sender = MetricsSender::builder().client(client).start(); + + let packet = receiver.recv().unwrap(); + let got = StatsdOutput::from_packets(iter::once(packet)); + assert_eq!(got, StatsdOutput::from([(b"foo:1|c".to_vec(), 1),])); + } +} diff --git a/osprey_coordinator/src/metrics/v2/statsd_output.rs b/osprey_coordinator/src/metrics/v2/statsd_output.rs new file mode 100644 index 0000000..a569ecc --- /dev/null +++ b/osprey_coordinator/src/metrics/v2/statsd_output.rs @@ -0,0 +1,61 @@ +use std::collections::HashMap; +use std::fmt::{self, Formatter}; +use std::ops::{Deref, DerefMut}; + +#[derive(Clone, Default, Eq, PartialEq)] +pub(crate) struct StatsdOutput(HashMap, u64>); + +impl StatsdOutput { + pub(crate) fn new() -> Self { + Self::default() + } + + pub(crate) fn from_packets(packets: impl IntoIterator>) -> Self { + packets.into_iter().fold(Self::new(), |mut m, packet| { + for line in packet.split(|&b| b == b'\n') { + if line.is_empty() { + continue; + } + m.entry(Vec::from(line)) + .and_modify(|n| *n += 1) + .or_insert(1); + } + m + }) + } +} + +impl Deref for StatsdOutput { + type Target = HashMap, u64>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for StatsdOutput { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl fmt::Debug for StatsdOutput { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let mut entries: Vec<(&[u8], u64)> = + self.0.iter().map(|(k, v)| (k.as_slice(), *v)).collect(); + entries.sort_by(|a, b| a.cmp(b)); + f.debug_map() + .entries( + entries + .into_iter() + .map(|(k, v)| (String::from_utf8_lossy(k), v)), + ) + .finish() + } +} + +impl From<[(Vec, u64); N]> for StatsdOutput { + fn from(arr: [(Vec, u64); N]) -> Self { + StatsdOutput(HashMap::from(arr)) + } +} diff --git a/osprey_coordinator/src/pigeon/grpc_client.rs b/osprey_coordinator/src/pigeon/grpc_client.rs new file mode 100644 index 0000000..1b74c04 --- /dev/null +++ b/osprey_coordinator/src/pigeon/grpc_client.rs @@ -0,0 +1,289 @@ +//! A wrapper client for automatically handling etcd based discovery with grpc. + +use crate::discovery::ServiceRegistration; +use crate::discovery::ServiceWatcher; +use crate::etcd::Client as EtcdClient; +use anyhow::Result; +use futures::future::try_join_all; +use futures::future::FutureExt; +use parking_lot::RwLock; +use std::collections::HashMap; +use std::fmt::Debug; +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; +use tonic::transport::Endpoint; + +use super::utils::ChunkedRequest; +use super::utils::Connector; +use super::utils::ScalarRequest; + +#[derive(Debug)] +struct Inner { + self_id: String, + num_replicas: usize, + // FIXME: Nothing ever remove bad clients from this + clients: RwLock>, + service_watcher: ServiceWatcher, + endpoint_options: EndpointOptions, +} + +#[derive(Clone, Debug)] +pub struct GrpcClient { + inner: Arc>, + peer_service: Option<&'static str>, +} + +impl GrpcClient +where + C: Clone + Connector, +{ + pub async fn new( + self_id: String, + service_name: String, + num_replicas: usize, + peer_service: Option<&'static str>, + ) -> Result { + let client = EtcdClient::from_etcd_peers()?; + let service_watcher = ServiceWatcher::watch(service_name, client.clone(), true).await?; + Self::new_with_watcher(self_id, num_replicas, service_watcher, peer_service).await + } + + pub async fn new_with_watcher( + self_id: String, + num_replicas: usize, + service_watcher: ServiceWatcher, + peer_service: Option<&'static str>, + ) -> Result { + Ok(GrpcClient { + inner: Arc::new(Inner { + self_id, + num_replicas, + clients: RwLock::new(HashMap::new()), + service_watcher, + endpoint_options: EndpointOptions::default(), + }), + peer_service, + }) + } + + pub fn with_endpoint_options(self, endpoint_options: EndpointOptions) -> Self { + GrpcClient { + inner: Arc::new(Inner { + self_id: self.inner.self_id.clone(), + num_replicas: self.inner.num_replicas, + clients: Default::default(), + service_watcher: self.inner.service_watcher.clone(), + endpoint_options, + }), + peer_service: self.peer_service, + } + } + + fn split_keys<'a, Key, KeyIterator>( + &self, + keys: KeyIterator, + tolerate_draining: bool, + ) -> Result, Vec)>> + where + Key: Clone + ToString + Debug + 'a, + KeyIterator: Iterator + 'a, + { + let mut chunks: HashMap, Vec)> = HashMap::new(); + for (i, key) in keys.enumerate() { + let service = &self.inner.service_watcher.select_key( + key.to_string(), + self.inner.num_replicas, + Some(&self.inner.self_id), + tolerate_draining, + 0, + )?; + let (index_chunk, key_chunk) = chunks + .entry(service.into()) + .or_insert((Vec::new(), Vec::new())); + index_chunk.push(i); + key_chunk.push(key.clone()); + } + Ok(chunks) + } + + fn get_client(&self, service_info: ServiceInfo) -> Result { + let key = &service_info.key; + + // Try grabbing client via read lock, just incase: + let client = self.inner.clients.read().get(key).cloned(); + if let Some(client) = client { + return Ok(client); + } + + // Acquire a write lock now, and try to create a client. + let mut clients = self.inner.clients.write(); + + // 1) Did anyone race us in creating the client, check first: + if let Some(client) = clients.get(key).cloned() { + return Ok(client); + } + + // 2) No one did, so let's create a client w/ connect_lazy, so that we don't keep this lock acquired, + // while the client connects. + let uri = service_info.uri(); + let endpoint = self.inner.endpoint_options.apply( + Endpoint::new(uri) + .map_err(|e| tonic::Status::unavailable(format!("invalid uri: {:?}", e)))?, + ); + + let client = C::connect_lazy(endpoint); + clients.insert(key.to_owned(), client.clone()); + + Ok(client) + } + + fn create_request(&self, new_request: Req) -> tonic::Request { + let mut grpc_request = tonic::Request::new(new_request); + if let Some(peer_service) = self.peer_service { + grpc_request.metadata_mut().insert( + "service", + peer_service + .parse() + .expect("conversion to MetadataValue failed"), + ); + } + grpc_request + } + + pub async fn chunked_unary_unary( + &self, + request: Req, + client_fn: F, + tolerate_draining: bool, + ) -> Result + where + K: Clone + ToString + Debug, + F: Fn(C, tonic::Request) -> FR, + FR: Future, tonic::Status>>, + Req: ChunkedRequest, + Resp: Into> + From>, + { + let chunks = self + .split_keys(request.iter_keys(), tolerate_draining) + .map_err(|e| tonic::Status::unavailable(format!("failed to split keys: {:?}", e)))?; + + let mut futs = Vec::with_capacity(chunks.len()); + + for (service_info, (indices, keys)) in chunks { + let client = self.get_client(service_info)?; + let new_request = request.with_keys(keys.clone()); + let fut = client_fn(client, self.create_request(new_request)) + // FIXME: We lose the metadata from the Response here :( + .map(|res| res.map(|resp| indices.into_iter().zip(resp.into_inner().into()))); + futs.push(fut); + } + let mut sorted: Vec<(usize, V)> = try_join_all(futs).await?.into_iter().flatten().collect(); + sorted.sort_by_key(|(i, _)| *i); + Ok(sorted + .into_iter() + .map(|(_, v)| v) + .collect::>() + .into()) + } + + pub async fn scalar_unary_unary, F, FR>( + &self, + request: Req, + client_fn: F, + tolerate_draining: bool, + instances_to_skip: u16, + ) -> Result + where + K: Clone + ToString + Debug, + F: Fn(C, tonic::Request) -> FR, + FR: Future, tonic::Status>>, + { + let inner = &self.inner; + let service = &inner + .service_watcher + .select_key( + request.get_key().to_string(), + inner.num_replicas, + Some(&inner.self_id), + tolerate_draining, + instances_to_skip, + ) + .map_err(|e| tonic::Status::unavailable(format!("failed get client. e={:?}", e)))?; + + let client = self.get_client(service.into())?; + let response = client_fn(client, self.create_request(request)) + .await? + .into_inner(); + Ok(response) + } + + pub async fn round_robin_unary_unary( + &self, + request: Req, + client_fn: F, + ) -> Result + where + F: Fn(C, tonic::Request) -> FR, + FR: Future, tonic::Status>>, + { + let service = &self + .inner + .service_watcher + .select() + .map_err(|e| tonic::Status::unavailable(format!("failed get client. e={:?}", e)))?; + + let client = self.get_client(service.into())?; + let response = client_fn(client, self.create_request(request)) + .await? + .into_inner(); + Ok(response) + } +} + +#[derive(Clone, Hash, PartialEq, Eq, Debug)] +struct ServiceInfo { + key: String, +} + +impl ServiceInfo { + fn uri(&self) -> String { + format!("http://{}", self.key) + } +} + +impl From<&ServiceRegistration> for ServiceInfo { + fn from(service: &ServiceRegistration) -> Self { + let grpc_port = match service.ports.get("grpc") { + Some(port) => *port, + None => service.port + 1, + }; + let key = match &service.ip { + Some(ip) => format!("{}:{}", ip, grpc_port), + None => format!("{}:{}", service.address, grpc_port), + }; + ServiceInfo { key } + } +} + +#[derive(Copy, Clone, Debug, Default)] +pub struct EndpointOptions { + pub timeout: Option, + pub connect_timeout: Option, + pub concurrency_limit: Option, +} + +impl EndpointOptions { + fn apply(&self, mut e: Endpoint) -> Endpoint { + if let Some(timeout) = self.timeout { + e = e.timeout(timeout); + } + if let Some(connect_timeout) = self.connect_timeout { + e = e.connect_timeout(connect_timeout); + } + if let Some(concurrency_limit) = self.concurrency_limit { + e = e.concurrency_limit(concurrency_limit); + } + e + } +} diff --git a/osprey_coordinator/src/pigeon/grpc_timeout.rs b/osprey_coordinator/src/pigeon/grpc_timeout.rs new file mode 100644 index 0000000..58650b4 --- /dev/null +++ b/osprey_coordinator/src/pigeon/grpc_timeout.rs @@ -0,0 +1,249 @@ +//! Copy of +//! https://github.com/hyperium/tonic/blob/v0.6.2/tonic/src/transport/service/grpc_timeout.rs + +use http::{HeaderMap, HeaderValue, Request}; +use pin_project::pin_project; +use std::{ + fmt, + future::Future, + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; +use tokio::time::Sleep; +use tower::Service; + +use super::option_pin::{OptionPin, OptionPinProj}; + +const GRPC_TIMEOUT_HEADER: &str = "grpc-timeout"; + +#[derive(Debug, Clone)] +pub(crate) struct GrpcTimeout { + inner: S, + server_timeout: Option, +} + +impl GrpcTimeout { + pub(crate) fn new(inner: S, server_timeout: Option) -> Self { + Self { + inner, + server_timeout, + } + } +} + +impl Service> for GrpcTimeout +where + S: Service>, + S::Error: Into, +{ + type Response = S::Response; + type Error = super::utils::Error; + type Future = ResponseFuture; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(Into::into) + } + + fn call(&mut self, req: Request) -> Self::Future { + let client_timeout = try_parse_grpc_timeout(req.headers()).unwrap_or_else(|e| { + tracing::trace!("Error parsing `grpc-timeout` header {:?}", e); + None + }); + + // Use the shorter of the two durations, if either are set + let timeout_duration = match (client_timeout, self.server_timeout) { + (None, None) => None, + (Some(dur), None) => Some(dur), + (None, Some(dur)) => Some(dur), + (Some(header), Some(server)) => { + let shorter_duration = std::cmp::min(header, server); + Some(shorter_duration) + } + }; + + ResponseFuture { + inner: self.inner.call(req), + sleep: timeout_duration + .map(tokio::time::sleep) + .map(|x| OptionPin::Some { inner: x }) + .unwrap_or(OptionPin::None), + } + } +} + +#[pin_project] +pub(crate) struct ResponseFuture { + #[pin] + inner: F, + #[pin] + sleep: OptionPin, +} + +impl Future for ResponseFuture +where + F: Future>, + E: Into, +{ + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + + if let Poll::Ready(result) = this.inner.poll(cx) { + return Poll::Ready(result.map_err(Into::into)); + } + + if let OptionPinProj::Some { inner: sleep } = this.sleep.project() { + futures::ready!(sleep.poll(cx)); + return Poll::Ready(Err(TimeoutExpired(()).into())); + } + + Poll::Pending + } +} + +const SECONDS_IN_HOUR: u64 = 60 * 60; +const SECONDS_IN_MINUTE: u64 = 60; + +/// Tries to parse the `grpc-timeout` header if it is present. If we fail to parse, returns +/// the value we attempted to parse. +/// +/// Follows the [gRPC over HTTP2 spec](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md). +fn try_parse_grpc_timeout( + headers: &HeaderMap, +) -> Result, &HeaderValue> { + match headers.get(GRPC_TIMEOUT_HEADER) { + Some(val) => { + let (timeout_value, timeout_unit) = val + .to_str() + .map_err(|_| val) + .and_then(|s| if s.is_empty() { Err(val) } else { Ok(s) })? + // `HeaderValue::to_str` only returns `Ok` if the header contains ASCII so this + // `split_at` will never panic from trying to split in the middle of a character. + // See https://docs.rs/http/0.2.4/http/header/struct.HeaderValue.html#method.to_str + // + // `len - 1` also wont panic since we just checked `s.is_empty`. + .split_at(val.len() - 1); + + // gRPC spec specifies `TimeoutValue` will be at most 8 digits + // Caping this at 8 digits also prevents integer overflow from ever occurring + if timeout_value.len() > 8 { + return Err(val); + } + + let timeout_value: u64 = timeout_value.parse().map_err(|_| val)?; + + let duration = match timeout_unit { + // Hours + "H" => Duration::from_secs(timeout_value * SECONDS_IN_HOUR), + // Minutes + "M" => Duration::from_secs(timeout_value * SECONDS_IN_MINUTE), + // Seconds + "S" => Duration::from_secs(timeout_value), + // Milliseconds + "m" => Duration::from_millis(timeout_value), + // Microseconds + "u" => Duration::from_micros(timeout_value), + // Nanoseconds + "n" => Duration::from_nanos(timeout_value), + _ => return Err(val), + }; + + Ok(Some(duration)) + } + None => Ok(None), + } +} + +/// Copy of [`tonic::transport::TimeoutExpired`]. +#[derive(Debug)] +pub(crate) struct TimeoutExpired(()); + +impl fmt::Display for TimeoutExpired { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Timeout expired") + } +} + +// std::error::Error only requires a type to impl Debug and Display +impl std::error::Error for TimeoutExpired {} + +#[cfg(test)] +mod tests { + use super::*; + + // Helper function to reduce the boiler plate of our test cases + fn setup_map_try_parse(val: Option<&str>) -> Result, HeaderValue> { + let mut hm = HeaderMap::new(); + if let Some(v) = val { + let hv = HeaderValue::from_str(v).unwrap(); + hm.insert(GRPC_TIMEOUT_HEADER, hv); + }; + + try_parse_grpc_timeout(&hm).map_err(|e| e.clone()) + } + + #[test] + fn test_hours() { + let parsed_duration = setup_map_try_parse(Some("3H")).unwrap().unwrap(); + assert_eq!(Duration::from_secs(3 * 60 * 60), parsed_duration); + } + + #[test] + fn test_minutes() { + let parsed_duration = setup_map_try_parse(Some("1M")).unwrap().unwrap(); + assert_eq!(Duration::from_secs(60), parsed_duration); + } + + #[test] + fn test_seconds() { + let parsed_duration = setup_map_try_parse(Some("42S")).unwrap().unwrap(); + assert_eq!(Duration::from_secs(42), parsed_duration); + } + + #[test] + fn test_milliseconds() { + let parsed_duration = setup_map_try_parse(Some("13m")).unwrap().unwrap(); + assert_eq!(Duration::from_millis(13), parsed_duration); + } + + #[test] + fn test_microseconds() { + let parsed_duration = setup_map_try_parse(Some("2u")).unwrap().unwrap(); + assert_eq!(Duration::from_micros(2), parsed_duration); + } + + #[test] + fn test_nanoseconds() { + let parsed_duration = setup_map_try_parse(Some("82n")).unwrap().unwrap(); + assert_eq!(Duration::from_nanos(82), parsed_duration); + } + + #[test] + fn test_header_not_present() { + let parsed_duration = setup_map_try_parse(None).unwrap(); + assert!(parsed_duration.is_none()); + } + + #[test] + #[should_panic(expected = "82f")] + fn test_invalid_unit() { + // "f" is not a valid TimeoutUnit + setup_map_try_parse(Some("82f")).unwrap().unwrap(); + } + + #[test] + #[should_panic(expected = "123456789H")] + fn test_too_many_digits() { + // gRPC spec states TimeoutValue will be at most 8 digits + setup_map_try_parse(Some("123456789H")).unwrap().unwrap(); + } + + #[test] + #[should_panic(expected = "oneH")] + fn test_invalid_digits() { + // gRPC spec states TimeoutValue will be at most 8 digits + setup_map_try_parse(Some("oneH")).unwrap().unwrap(); + } +} diff --git a/osprey_coordinator/src/pigeon/health.rs b/osprey_coordinator/src/pigeon/health.rs new file mode 100644 index 0000000..5f2cfb0 --- /dev/null +++ b/osprey_coordinator/src/pigeon/health.rs @@ -0,0 +1,197 @@ +use futures::stream::{Stream, StreamExt}; +use std::borrow::Cow; +use std::pin::Pin; +use std::time::Duration; +use std::{collections::HashMap, sync::Arc}; +use tonic::{Request, Response, Status}; +use tonic_health::pb::{ + health_check_response::ServingStatus, health_server::Health, HealthCheckRequest, + HealthCheckResponse, +}; + +const HEALTH_CHECK_ALL_PSEUDO_SERVICE: &str = "discord.v1.HealthCheckAll"; + +pub trait HealthChecker: Send + Sync + 'static { + fn is_healthy(&self) -> bool; +} + +impl bool + Send + Sync + 'static> HealthChecker for F { + fn is_healthy(&self) -> bool { + self() + } +} + +/// For any type `T` implementing `HealthChecker`, +/// `Arc` also implements `HealthChecker`. +/// This permits passing `Arc` anywhere that accepts an `impl HealthChecker`, e.g. `fn foo(x: impl HealthChecker)` +impl HealthChecker for Arc { + fn is_healthy(&self) -> bool { + self.as_ref().is_healthy() + } +} + +/// A server readiness check that always returns healthy. +#[derive(Clone, Debug, Eq, PartialEq, Hash, Default)] +pub struct AlwaysHealthy {} + +impl HealthChecker for AlwaysHealthy { + fn is_healthy(&self) -> bool { + true + } +} + +#[derive(Clone)] +pub(crate) struct HealthServer { + callbacks: Arc, Vec>>>, + shutting_down: tokio::sync::watch::Receiver, +} + +impl HealthServer { + pub(crate) fn new( + callbacks: impl Into, Vec>>>>, + ) -> (Self, ShutdownSender) { + let (tx, rx) = tokio::sync::watch::channel(false); + ( + HealthServer { + callbacks: callbacks.into(), + shutting_down: rx, + }, + ShutdownSender { inner: tx }, + ) + } + + fn is_healthy( + checks: &[Box], + shutting_down: &tokio::sync::watch::Receiver, + ) -> bool { + !*shutting_down.borrow() && checks.iter().all(|c| c.is_healthy()) + } + + fn is_healthy_and_update( + checks: &[Box], + shutting_down: &mut tokio::sync::watch::Receiver, + ) -> bool { + !*shutting_down.borrow_and_update() && checks.iter().all(|c| c.is_healthy()) + } + + fn is_all_services_healthy(&self) -> bool { + if *self.shutting_down.borrow() { + return false; + } + + for checks in self.callbacks.values() { + for check in checks { + if !check.is_healthy() { + return false; + } + } + } + + true + } +} + +#[tonic::async_trait] +impl Health for HealthServer { + async fn check( + &self, + request: Request, + ) -> Result, Status> { + let service_name = request.get_ref().service.as_str(); + if service_name.is_empty() { + // Liveness check should always return Serving. + return Ok(Response::new(bool_to_health_response(true))); + } + + if service_name == HEALTH_CHECK_ALL_PSEUDO_SERVICE { + return Ok(Response::new(bool_to_health_response( + self.is_all_services_healthy(), + ))); + } + + match self.callbacks.get(service_name) { + Some(checks) => Ok(Response::new(bool_to_health_response( + HealthServer::is_healthy(checks, &self.shutting_down), + ))), + None => Err(not_found_error(service_name)), + } + } + + type WatchStream = + Pin> + Send + 'static>>; + + async fn watch( + &self, + request: Request, + ) -> Result, Status> { + let service_name = request.get_ref().service.to_string(); + if service_name.is_empty() { + // Liveness check should always return Serving, + // and then stay open forever. + return Ok(Response::new(Box::pin( + futures::stream::once(async { Ok(bool_to_health_response(true)) }) + .chain(futures::stream::pending()), + ))); + } + let HealthServer { + callbacks, + mut shutting_down, + } = self.clone(); + let initial_value = match callbacks.get(service_name.as_str()) { + Some(checks) => HealthServer::is_healthy_and_update(checks, &mut shutting_down), + None => { + return Err(not_found_error(&service_name)); + } + }; + + let output = async_stream::try_stream! { + yield bool_to_health_response(initial_value); + + let checks = &callbacks[service_name.as_str()]; + let mut interval = tokio::time::interval(Duration::from_millis(1000)); + let mut prev_value = initial_value; + loop { + tokio::select!{ + _ = interval.tick() => {}, + _ = shutting_down.changed() => {}, + }; + let new_value = HealthServer::is_healthy_and_update(checks, &mut shutting_down); + if new_value != prev_value { + yield bool_to_health_response(new_value); + prev_value = new_value; + } + } + }; + + Ok(Response::new(Box::pin(output) as Self::WatchStream)) + } +} + +pub(crate) struct ShutdownSender { + inner: tokio::sync::watch::Sender, +} + +impl ShutdownSender { + pub(crate) fn notify(&self) { + // TODO: Replace with send_if_modified once Tokio is upgraded. + if !*self.inner.borrow() { + let _ = self.inner.send(true); + } + } +} + +fn bool_to_health_response(healthy: bool) -> HealthCheckResponse { + HealthCheckResponse { + status: if healthy { + ServingStatus::Serving as i32 + } else { + ServingStatus::NotServing as i32 + }, + } +} + +fn not_found_error(service_name: &str) -> Status { + Status::not_found(format!("service {} not registered", service_name)) +} + +// TODO copy over tests diff --git a/osprey_coordinator/src/pigeon/instrumented_connection.rs b/osprey_coordinator/src/pigeon/instrumented_connection.rs new file mode 100644 index 0000000..0c29542 --- /dev/null +++ b/osprey_coordinator/src/pigeon/instrumented_connection.rs @@ -0,0 +1,181 @@ +use crate::metrics::string_interner::{InternResult, StringInterner}; +use crate::metrics::{MetricsEmitter, TagKey}; +use hyper::server::accept::Accept; +use hyper::server::conn::{AddrIncoming, AddrStream}; +use lazy_static::lazy_static; +use pin_project::{pin_project, pinned_drop}; +use std::os::fd::AsRawFd; +use std::pin::Pin; +use std::sync::Arc; +use std::task::Poll; +use tokio::io::{AsyncRead, AsyncWrite}; + +use crate::metrics::v2::{Counter, Gauge}; + +/// Define the tags for our connection-accept counter. +/// We currently only track the status and the address we are bound to +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, TagKey)] +struct AcceptRateKey { + #[tag(name = "status")] + status: &'static str, + #[tag(name = "local_address")] + local_address: InternResult, +} + +/// Define the tags for our connection gauge. +/// We only track the local bind address +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, TagKey)] +struct ConnectionKey { + #[tag(name = "local_address")] + local_address: InternResult, +} + +/// A structure defining all of the metrics we export from our [`InstrumentedConnection`] +/// and [`InstrumentedIncoming`]. These metrics are at a level below grpc, and even hyper. +/// This is purely a way to see what the underlying tcp is doing. +#[derive(Debug, MetricsEmitter)] +struct ConnectionMetrics { + #[metric(name = "pigeon.server.connection_count")] + connection_count: Gauge, + #[metric(name = "pigeon.server.connection_rate")] + connection_rate: Counter, +} + +lazy_static! { + static ref CONNECTION_METRICS: Arc = { + let metrics = Arc::new(ConnectionMetrics { + connection_count: Default::default(), + connection_rate: Default::default(), + }); + crate::metrics::v2::register_emitter(metrics.clone()).emit_forever(); + metrics + }; + // NOTE: load bearing assertion that we won't bind on more than 16 addresses + static ref LOCAL_ADDR_INTERNER: StringInterner = StringInterner::new(16); +} + +/// An InstrumentedConnection is a very simple wrapper around hyper's [`AddrStream`] +/// All it adds on top is a [`Gauge`] tracking the number of connections from a given +/// listening socket. +#[pin_project(PinnedDrop)] +pub struct InstrumentedConnection { + metrics_key: ConnectionKey, + #[pin] + inner: AddrStream, +} + +impl InstrumentedConnection { + pub fn new(inner: AddrStream, local_addr: InternResult) -> Self { + let metrics_key = ConnectionKey { + local_address: local_addr, + }; + CONNECTION_METRICS.connection_count.incr(metrics_key, 1_u64); + Self { metrics_key, inner } + } +} + +#[pinned_drop] +impl PinnedDrop for InstrumentedConnection { + fn drop(self: Pin<&mut Self>) { + CONNECTION_METRICS + .connection_count + .decr(self.metrics_key, 1_u64); + } +} + +impl AsRawFd for InstrumentedConnection { + fn as_raw_fd(&self) -> std::os::fd::RawFd { + self.inner.as_raw_fd() + } +} + +impl AsyncRead for InstrumentedConnection { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + self.project().inner.poll_read(cx, buf) + } +} + +impl AsyncWrite for InstrumentedConnection { + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + + fn poll_write( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + self.project().inner.poll_write(cx, buf) + } + + fn poll_flush( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.project().inner.poll_flush(cx) + } + + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.project().inner.poll_shutdown(cx) + } + + fn poll_write_vectored( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + bufs: &[std::io::IoSlice<'_>], + ) -> std::task::Poll> { + self.project().inner.poll_write_vectored(cx, bufs) + } +} + +/// We provider a wrapper around hyper's [`AddrIncoming`] to instrument the rate of accepts +/// and to wrap all resulting [`AddrStream`] instances with [`InstrumentedConnection`] +#[pin_project] +pub struct InstrumentedIncoming { + local_addr: InternResult, + #[pin] + inner: AddrIncoming, +} + +impl InstrumentedIncoming { + pub fn new(inner: AddrIncoming) -> Self { + let local_addr = LOCAL_ADDR_INTERNER.intern(inner.local_addr().to_string()); + Self { local_addr, inner } + } +} + +impl Accept for InstrumentedIncoming { + type Conn = InstrumentedConnection; + type Error = std::io::Error; + fn poll_accept( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll>> { + let laddr = self.local_addr; + match self.project().inner.poll_accept(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(None) => Poll::Ready(None), + Poll::Ready(Some(Ok(conn))) => { + CONNECTION_METRICS.connection_rate.incr(AcceptRateKey { + local_address: laddr, + status: "ok", + }); + Poll::Ready(Some(Ok(InstrumentedConnection::new(conn, laddr)))) + } + Poll::Ready(Some(Err(e))) => { + CONNECTION_METRICS.connection_rate.incr(AcceptRateKey { + local_address: laddr, + status: "error", + }); + Poll::Ready(Some(Err(e))) + } + } + } +} diff --git a/osprey_coordinator/src/pigeon/middleware.rs b/osprey_coordinator/src/pigeon/middleware.rs new file mode 100644 index 0000000..859d314 --- /dev/null +++ b/osprey_coordinator/src/pigeon/middleware.rs @@ -0,0 +1,234 @@ +//! This file is mostly a copy of +//! https://github.com/hyperium/tonic/blob/v0.6.2/tonic/src/transport/server/mod.rs#L757-L902 + +use anyhow::Result; +use http::{Request, Response}; +use hyper::Body; +use opentelemetry::trace::TraceContextExt; +use opentelemetry_http::HeaderExtractor; +use pin_project::pin_project; +use std::future::{self, Future}; +use std::pin::Pin; +use tonic::body::BoxBody; +use tower::util::BoxService; +use tower::{Service as TowerService, ServiceBuilder}; +use tracing::{debug_span, field, info_span}; +use tracing_opentelemetry::OpenTelemetrySpanExt; + +use super::grpc_timeout::GrpcTimeout; +use super::instrumented_connection::InstrumentedConnection; +use super::recover_error::RecoverError; +use super::router::Router; +use super::utils::Error; + +// NOTE: When a field is added here, the value must be pre-set also in the the `info_span!(...)` +// declaration within `extract_span`. +static HEADERS_TO_RECORD_AS_SPAN_FIELDS: &[(&str, &str)] = &[ + ("service", "peer_service"), + ("service-version", "peer_service_version"), +]; + +// Health gRPC Service Name and Package so we can set them as debug spans +const GRPC_HEALTH_PACKAGE: &str = "grpc.health.v1"; +const GRPC_HEALTH_SERVICE: &str = "Health"; + +/// Factory for [`TracedService`]. +pub(crate) struct ServiceFactory { + pub(crate) router: Router, +} + +impl<'a, S> TowerService<&'a InstrumentedConnection> for ServiceFactory +where + S: TowerService, Response = Response> + Clone + Send + 'static, + S::Future: Send + 'static, + S::Error: Into, +{ + type Response = BoxService, Response, Error>; + type Error = Error; + type Future = future::Ready>; + + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn call(&mut self, _socket: &'a InstrumentedConnection) -> Self::Future { + let svc = self.router.clone(); + let svc = ServiceBuilder::new() + .layer_fn(RecoverError::new) + .layer_fn(|s| GrpcTimeout::new(s, None)) + .service(svc); + let svc = ServiceBuilder::new() + .layer(BoxService::layer()) + .service(TracedService { inner: svc }); + future::ready(Ok(svc)) + } +} + +type BoxErrorBody = http_body::combinators::UnsyncBoxBody; + +/// Service that starts a trace span. +#[derive(Clone)] +struct TracedService { + inner: S, +} + +impl TowerService> for TracedService +where + S: TowerService, Response = Response>, + S::Error: Into, + ResBody: http_body::Body + Send + 'static, + ResBody::Error: Into, +{ + type Response = Response; + type Error = Error; + type Future = TracedServiceFuture; + + fn poll_ready( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.inner.poll_ready(cx).map_err(Into::into) + } + + fn call(&mut self, req: Request) -> Self::Future { + let span = extract_span(&req); + + TracedServiceFuture { + inner: self.inner.call(req), + span, + } + } +} + +#[pin_project] +struct TracedServiceFuture { + #[pin] + inner: F, + span: tracing::Span, +} + +impl Future for TracedServiceFuture +where + F: Future, E>>, + E: Into, + ResBody: http_body::Body + Send + 'static, + ResBody::Error: Into, +{ + type Output = Result, Error>; + + fn poll( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + use http_body::Body; + let this = self.project(); + let _guard = this.span.enter(); + + let response: Response = + futures::ready!(this.inner.poll(cx)).map_err(Into::into)?; + let response = response.map(|body| body.map_err(Into::into).boxed_unsync()); + std::task::Poll::Ready(Ok(response)) + } +} + +fn extract_span(request: &Request) -> tracing::Span { + let headers = request.headers(); + let extractor = HeaderExtractor(headers); + let parent_cx = opentelemetry::global::get_text_map_propagator(|prop| prop.extract(&extractor)); + + let (pkg, svc, method) = parse_path(request.uri().path()); + + // `otel.name` overrides the static span name ("grpc.request") we pass below. + // this is good for otel, but for datadog we still need the primary operation name, + // which we pass as `dd.operation = "grpc.request"`. + let otel_name = match (svc, method) { + (Some(svc), Some(method)) => Some(format!("{svc}/{method}")), + _ => None, + }; + + let tracing_span = match (pkg, svc) { + // Set gRPC Health requests to debug level + (Some(p), Some(s)) if p == GRPC_HEALTH_PACKAGE && s == GRPC_HEALTH_SERVICE => { + debug_span!( + "grpc.request", + otel.name = field::Empty, + dd.operation = field::Empty, + peer_service = field::Empty, + peer_service_version = field::Empty, + rpc.grpc.path = request.uri().path(), + rpc.grpc.package = p, + rpc.service = s, + rpc.method = method, + ) + } + _ => info_span!( + "grpc.request", + otel.name = field::Empty, + dd.operation = field::Empty, + peer_service = field::Empty, + peer_service_version = field::Empty, + rpc.grpc.path = request.uri().path(), + rpc.grpc.package = pkg, + rpc.service = svc, + rpc.method = method, + ), + }; + + // Note: The use_dd_operation feature was in the tracing_utils crate + // For now, we'll always record these fields + tracing_span.record("otel.name", otel_name); + tracing_span.record("dd.operation", "grpc.request"); + + // Only set a parent span if it is "valid", else treat this span as the root. + // A parent span is "valid" if it contains a trace id and span id. + // This check fixes an issue where requests directly to a service without span headers were creating + // "invalid" parent spans, causing the child span to be dropped. + if parent_cx.span().span_context().is_valid() { + tracing_span.set_parent(parent_cx); + } + + for (header_key, field_key) in HEADERS_TO_RECORD_AS_SPAN_FIELDS { + if let Some(header_value) = headers.get(*header_key) { + if let Ok(header_value_str) = header_value.to_str() { + tracing_span.record(*field_key, &header_value_str); + } + } + } + + tracing_span +} + +fn str_is_empty(s: &str) -> Option<&str> { + if s.is_empty() { + None + } else { + Some(s) + } +} + +// Parses the gRPC Path and returns the (package, service, method). +fn parse_path(path: &str) -> (Option<&str>, Option<&str>, Option<&str>) { + let tail = match path.strip_prefix('/') { + Some(tail) => tail, + None => return (None, None, None), + }; + + let (service_with_pkg, method) = match tail.split_once('/') { + Some((a, b)) => (str_is_empty(a), str_is_empty(b)), + None => return (None, None, None), + }; + + let (pkg, svc) = match service_with_pkg { + Some(svc_pkg) => match svc_pkg.rsplit_once(".") { + Some((a, b)) => (str_is_empty(a), str_is_empty(b)), + // Assume pkg is optional, and just treat entire left side as svc + None => (None, Some(svc_pkg)), + }, + None => (None, None), + }; + + (pkg, svc, method) +} diff --git a/osprey_coordinator/src/pigeon/mod.rs b/osprey_coordinator/src/pigeon/mod.rs new file mode 100644 index 0000000..a9bc96d --- /dev/null +++ b/osprey_coordinator/src/pigeon/mod.rs @@ -0,0 +1,589 @@ +use crate::discovery::{ServiceAnnouncer, ServiceRegistration}; +use crate::etcd::Client as EtcdClient; +use crate::metrics::v2::{MetricsSender, SendMetricsError}; +use crate::pigeon::health::{HealthChecker, HealthServer}; +use crate::pigeon::pigeon_metrics::MetricsLayer; +use crate::signals::{self, exit_signal}; +use anyhow::Result; +use futures::future::{MapErr, Shared}; +use futures::stream::{FuturesUnordered, StreamExt}; +use futures::{self, FutureExt, TryFutureExt}; +use http::{Request, Response}; +use hyper::server::conn::AddrIncoming; +use hyper::Body; +use instrumented_connection::InstrumentedIncoming; +use std::future::{self, Future}; +use std::iter; +use std::net::Ipv4Addr; +use std::net::SocketAddr; +use std::pin::Pin; +use std::{ + borrow::{Borrow, Cow}, + sync::{atomic::AtomicUsize, Arc}, +}; +use std::{collections::HashMap, sync::atomic::Ordering}; +use std::{env, time::Duration}; +use tokio::task::{spawn, JoinError}; +use tokio::time::sleep; +use tokio::{sync::oneshot, time::Instant}; +use tonic::body::BoxBody; +use tonic_health::pb::health_server::HealthServer as HealthProtoServer; +use tower::util::BoxCloneService; +use tower::{Layer, Service as TowerService, ServiceExt}; +use tracing::{debug, info, warn}; + +pub mod grpc_client; +mod grpc_timeout; +mod health; +mod instrumented_connection; +mod middleware; +mod option_pin; +mod pigeon_metrics; +mod recover_error; +mod router; +mod status; +mod utils; + +use self::utils::Error; +use router::Router; + +pub use self::utils::Connector; +pub use tonic::server::NamedService; + +pub async fn serve( + grpc_service: GS, + service_name: &'static str, + service_port: u16, + announce_delay: Duration, +) -> Result<()> +where + GS: GrpcService + NamedService + Clone + Send + 'static, + GS::Future: Send + 'static, + GS::Error: Into> + Send, +{ + Server::new(grpc_service, service_name, service_port) + .with_standard_registration( + service_name, + env::var("POD_IP").expect("`POD_IP needs to be set").into(), + ) + .with_encoded_file_descriptor_set(crate::proto::PB_DESCRIPTOR_BYTES) + .with_announce_delay(Some(announce_delay)) + .with_shutdown(signals::exit_signal()) + .serve() + .await +} + +/// The TCP keep-alive ping interval for the GRPC servers that pigeon runs. +const TCP_KEEPALIVE_DURATION: Duration = Duration::from_secs(300); + +/// Copy of +/// [`tonic_reflection::proto::server_reflection_server::ServerReflectionServer::NAME`] +/// for visibility reasons. +const REFLECTION_SERVICE_NAME: &str = "grpc.reflection.v1alpha.ServerReflection"; + +pub trait GrpcService +where + Self: TowerService, Response = Response>, + Self::Future: Send + 'static, + Self::Error: Into + Send, +{ +} + +impl GrpcService for TS +where + TS: TowerService, Response = Response>, + TS::Future: Send + 'static, + TS::Error: Into + Send, +{ +} + +/// Defines the configuration of how the server will shut down after receiving an exit signal. +/// Set using [`Server::with_shutdown_config`]. +pub enum ShutdownConfig { + /// If requests have not stopped in `max_delay`, then the server will shut down anyways. + WaitForRequestsToStop { + /// The maximum amount of time to wait for requests to stop. + max_delay: Duration, + /// The amount of time to wait for no requests to be received before shutting down. + acquiesce_duration: Duration, + }, +} + +struct DiscoveryConfig { + /// The delay to start announcing after the GRPC service is started. + announce_delay: Option, + /// The delay to deregister from service discovery after receiving an exit signal. + deregister_delay: Option, + /// The optional etcd-client struct for handling registration + etcd_client: Option, + /// Whether to wait for an existing key to be freed. Defaults to `false`. Timeout is required. + wait_for_free_key_timeout: Option, + /// Service information for registering in discovery. + registrations: Vec, +} + +/// A Server that handles the common logic needed to play well with +/// other services in our ecosystem. +pub struct Server { + address: SocketAddr, + /// If true, will not execute announce_delay or shutdown_config. Defaults to false. + debug: bool, + + /// We gate all of our etcd discovery based config behind this structure to make the code cleaner + discovery_config: DiscoveryConfig, + + /// Configuration defining how to shut down the server after receiving an exit signal. Ignored if debug is true. + shutdown_config: ShutdownConfig, + /// Metrics prefix. + metrics_prefix: String, + + /// gRPC services that do business logic. + grpc_services: HashMap<&'static str, (BoxGrpcService, Vec>)>, + /// Protobuf reflection data. + /// If empty, don't serve reflection. + encoded_file_descriptor_sets: Vec>, + + /// A signal to shutdown on. + shutdown: Pin + Send>>, + + /// A callback to be called synchronously after the server has received + /// an interrupt signal but BEFORE being deregistered from service discovery + pre_unannounce_callback: + Option Pin + Send>> + Send>>, + + allow_http1_requests: bool, + + /// Configures the maximum number of pending reset streams allowed before a GOAWAY will be sent. + /// This is currently being used for testing and is not recommended to be set in production. + /// See https://github.com/hyperium/h2/pull/668 for more details. + http2_max_pending_accept_reset_streams: Option, +} + +impl Server { + /// Create a server for the given gRPC server stub. + /// The returned `Server` will not announce to etcd + /// unless [`with_registration`] or [`with_standard_registration`] are called. + pub fn new(service: GS, metrics_prefix: impl Into, port: u16) -> Server + where + GS: GrpcService + NamedService + Clone + Send + 'static, + GS::Future: Send + 'static, + GS::Error: Into + Send, + { + let address = SocketAddr::new(Ipv4Addr::UNSPECIFIED.into(), port); + + let mut grpc_services = HashMap::new(); + grpc_services.insert(GS::NAME, (box_service(service), Vec::new())); + + let discovery_config = DiscoveryConfig { + announce_delay: Some(Duration::from_secs(5)), + deregister_delay: None, + etcd_client: None, + registrations: Vec::new(), + wait_for_free_key_timeout: None, + }; + + Self { + address, + debug: false, + shutdown_config: ShutdownConfig::WaitForRequestsToStop { + max_delay: Duration::from_secs(300), + acquiesce_duration: Duration::from_secs(15), + }, + discovery_config, + metrics_prefix: metrics_prefix.into(), + grpc_services, + encoded_file_descriptor_sets: Vec::new(), + shutdown: Box::pin(future::pending()), + pre_unannounce_callback: None, + allow_http1_requests: false, + http2_max_pending_accept_reset_streams: None, + } + } + + /// Adds an standard etcd service registration to announce while serving. + pub fn with_standard_registration( + self, + registration_name: impl Into, + hostname: Option>, + ) -> Self { + let hostname = match &hostname { + Some(hostname) => hostname.as_ref().trim().to_string(), + None => hostname::get() + .map(|os| os.to_string_lossy().into_owned()) + .unwrap_or_else(|_| "localhost".to_string()), + }; + + let registration = + ServiceRegistration::builder(registration_name.into(), self.address.port()) + .with_grpc_port(self.address.port()) + .with_address_override(hostname) + .build(); + + self.with_registration(registration) + } + + /// Adds an etcd service registration to announce while serving. + pub fn with_registration(mut self, service: ServiceRegistration) -> Self { + self.discovery_config.registrations.push(service); + self + } + + /// Sets a signal that the service should shutdown on. Defaults to `None`. + pub fn with_shutdown(mut self, shutdown: NS) -> Self + where + NS: Future + Send + 'static, + { + self.shutdown = Box::pin(shutdown); + self + } + + /// Sets the amount of time between grpc service start up and when the announcer + /// starts. Use this setting if your service has a warm up. + /// Defaults to 5 seconds. + pub fn with_announce_delay(mut self, announce_delay: Option) -> Self { + self.discovery_config.announce_delay = announce_delay; + self + } + + /// Adds a Protocol Buffer file descriptor set (reflection information) to be served. + /// Calling this at least once will enable [gRPC Server Reflection]. + /// + /// [gRPC Server Reflection]: https://github.com/grpc/grpc/blob/master/doc/server-reflection.md + pub fn with_encoded_file_descriptor_set( + mut self, + fd_set: impl Into>, + ) -> Self { + self.encoded_file_descriptor_sets.push(fd_set.into()); + self + } + + /// Starts the server and runs until the service stops or an interrupt + /// signal is received. + /// + /// This will do the following: + /// 1. Handle registering/deregistring the service with Discovery. + /// 2. Register metrics for the GRPC service. + /// 3. Start the GRPC service. + /// 4. Listen for a stop signal, including ctrl+c and other system interrupts. + /// 5. Wait for the GRPC service to gracefully stop + pub async fn serve(self) -> Result<()> { + self.serve_inner().await + } + + async fn serve_inner(self) -> Result<()> { + let Server { + address, + debug, + shutdown_config, + metrics_prefix, + discovery_config, + grpc_services, + shutdown, + pre_unannounce_callback, + encoded_file_descriptor_sets, + allow_http1_requests, + http2_max_pending_accept_reset_streams, + } = self; + info!("Starting service on {}", address); + + // Start sending metrics if the caller has not started doing so. + // The thread will be shut down when the future is resolved or dropped. + let sender_result = MetricsSender::builder().start(); + + let request_counter = Arc::new(AtomicUsize::new(0)); + + let _sender = match sender_result { + Ok(sender) => { + debug!("Started sending statsd metrics"); + Some(sender) + } + Err(SendMetricsError::AlreadySending) => None, + Err(err) => return Err(err.into()), + }; + let (grpc_services, health_checks): (HashMap<_, _>, HashMap<_, _>) = grpc_services + .into_iter() + .map(|(k, (srv, checks))| ((k, srv), (Cow::Borrowed(k), checks))) + .unzip(); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + + let (health_server, health_shutdown_tx) = HealthServer::new(health_checks); + + let request_counter_for_server = request_counter.clone(); + let error_nullifier: ErrorNullifier = |_| (); + let join_handle = spawn(async move { + let result = listen_and_serve( + &address, + &metrics_prefix, + grpc_services, + HealthProtoServer::new(health_server), + encoded_file_descriptor_sets, + shutdown_rx, + allow_http1_requests, + http2_max_pending_accept_reset_streams, + request_counter_for_server, + ) + .await; + warn!("GRPC server ({:?}) stopped serving. {:?}", address, result); + }) + .map_err(error_nullifier) + .shared(); // Make join_handle sharable, as we will try to await multiple times. + let grpc_handle = GrpcHandle::builder( + shutdown_tx, + join_handle.clone(), + request_counter, + shutdown_config, + ); + + let announcers: Vec = { + if !debug { + if let Some(announce_delay) = discovery_config.announce_delay { + // Give the service time to warm up before announcing. + sleep(announce_delay).await; + } + } + if discovery_config.registrations.is_empty() { + Vec::new() + } else { + let etcd_client = discovery_config + .etcd_client + .map(Result::Ok) + .unwrap_or_else(|| EtcdClient::from_etcd_peers())?; + discovery_config + .registrations + .into_iter() + .map(|service| { + ServiceAnnouncer::builder(service, etcd_client.clone()) + .wait_for_free_key(discovery_config.wait_for_free_key_timeout) + .start() + }) + .collect() + } + }; + + let exit_signal = exit_signal(); + + tokio::select! { + _ = shutdown => warn!("Shutdown received."), + _ = join_handle.clone() => warn!("Down received."), + _ = exit_signal => warn!("Graceful exit signal received."), + }; + + if !debug { + if let Some(deregister_delay) = discovery_config.deregister_delay { + // Give service time to drain while still being discoverable. + sleep(deregister_delay).await; + } + } + + info!("Stopping services."); + health_shutdown_tx.notify(); + + // Execute pre_unannounce_callback if it exists before the service is de-registered (or marked as draining) + // from service discovery. + if let Some(callback) = pre_unannounce_callback { + (callback)().await; + } + let announcer_stops: FuturesUnordered<_> = + announcers.into_iter().map(|a| a.stop()).collect(); + announcer_stops.for_each(|_| future::ready(())).await; + if !debug { + grpc_handle.shutdown().await?; + } else { + grpc_handle.stop().await?; + } + info!("Services stopped."); + Ok(()) + } +} + +async fn listen_and_serve<'a, THealthServer>( + address: &'a SocketAddr, + metrics_prefix: &'a str, + grpc_services: HashMap<&'static str, BoxGrpcService>, + health_server: THealthServer, + encoded_file_descriptor_sets: Vec>, + shutdown_rx: oneshot::Receiver<()>, + allow_http1_requests: bool, + http2_max_pending_accept_reset_streams: Option, + request_counter: Arc, +) -> Result<()> +where + THealthServer: GrpcService + NamedService + Clone + Send + 'static, + THealthServer::Future: Send + 'static, + THealthServer::Error: Into + Send, +{ + let reflection_service = if encoded_file_descriptor_sets.is_empty() { + None + } else { + let mut reflection_builder = tonic_reflection::server::Builder::configure(); + for encoded_fd_set in &encoded_file_descriptor_sets { + reflection_builder = + reflection_builder.register_encoded_file_descriptor_set(encoded_fd_set.borrow()); + } + Some(reflection_builder.build()?) + }; + + let service_map: HashMap<&'static str, BoxGrpcService> = grpc_services + .into_iter() + .map(|(name, grpc_service)| { + ( + name, + box_service( + MetricsLayer::server(metrics_prefix) + .with_request_counter(request_counter.clone()) + .layer(grpc_service) + .map_response(|resp| resp.map(BoxBody::new)), + ), + ) + }) + .chain(iter::once(( + THealthServer::NAME, + box_service( + MetricsLayer::server(metrics_prefix) + .layer(health_server) + .map_response(|resp| resp.map(BoxBody::new)), + ), + ))) + .chain( + reflection_service + .map(|s| (REFLECTION_SERVICE_NAME, box_service(s))) + .into_iter(), + ) + .collect(); + + let mut incoming = AddrIncoming::bind(address)?; + incoming + .set_keepalive(Some(TCP_KEEPALIVE_DURATION)) + .set_nodelay(true); + let server = hyper::Server::builder(InstrumentedIncoming::new(incoming)) + .http2_only(!allow_http1_requests) + .http2_max_pending_accept_reset_streams(http2_max_pending_accept_reset_streams); + + server + .serve(middleware::ServiceFactory { + router: Router { + services: service_map, + }, + }) + .with_graceful_shutdown(shutdown_rx.map(|_| ())) + .await?; + + Ok(()) +} + +type BoxGrpcService = BoxCloneService, Response, Error>; + +fn box_service(inner: S) -> BoxGrpcService +where + S: TowerService, Response = Response, Error = E> + + Clone + + Send + + 'static, + S::Future: Send + 'static, + E: Into + 'static, +{ + BoxCloneService::new(inner.map_err(Into::into)) +} + +type ErrorNullifier = fn(JoinError) -> (); +type SharedErrorNullifiedJoinHandle = Shared, ErrorNullifier>>; + +struct GrpcHandleInner { + shutdown_tx: oneshot::Sender<()>, + join_handle: SharedErrorNullifiedJoinHandle, +} + +/// A [`GrpcHandle`] can stop a GRPC service when dropped or when [`GrpcHandle::stop`] is called. +/// +/// It can also see the number of requests that have been received by the GRPC service during it's +/// lifetime by calling [`GrpcHandle::get_request_count`]. +pub struct GrpcHandle { + inner: Option, + request_counter: Arc, + shutdown_config: ShutdownConfig, +} + +impl GrpcHandle { + pub fn builder( + shutdown_tx: oneshot::Sender<()>, + join_handle: SharedErrorNullifiedJoinHandle, + request_counter: Arc, + shutdown_config: ShutdownConfig, + ) -> GrpcHandle { + Self { + inner: Some(GrpcHandleInner { + shutdown_tx, + join_handle, + }), + request_counter, + shutdown_config, + } + } + + /// Gets the number of requests that have been received by the GRPC service. + pub fn request_count(&self) -> usize { + self.request_counter.load(Ordering::Relaxed) + } + + /// Stops the GRPC service in accordance to the [`ShutdownConfig`] supplied to the [`Server`] builder. + pub async fn shutdown(self) -> Result<()> { + match self.shutdown_config { + ShutdownConfig::WaitForRequestsToStop { + max_delay, + acquiesce_duration, + } => { + // Future that waits for requests to stop, returning the time it took for requests to stop. + let stopped_future = async { + let mut prev_request_count = self.request_count(); + let start_time = Instant::now(); + + // This tracks the time at when we last observed an incoming request. + let mut acequiesce_start = start_time; + + // Poll the request counter, detecting a period where there are no requests + // for at least `acquiesce_duration`. + loop { + sleep(Duration::from_millis(100)).await; + + let current_request_count = self.request_count(); + if prev_request_count != current_request_count { + prev_request_count = current_request_count; + acequiesce_start = Instant::now(); + } else { + if acequiesce_start.elapsed() > acquiesce_duration { + return start_time.elapsed(); + } + } + } + }; + + info!("Waiting for requests to stop, will wait for no requests to be received for {:?}, or at most {:?} before stopping.", acquiesce_duration, max_delay); + tokio::select! { + elapsed = stopped_future => { + info!("Requests have stopped in {:?}. Shutting down.", elapsed); + }, + _ = sleep(max_delay) => { + info!("Requests have not acquiesced in {:?}. Shutting down anyways.", max_delay); + }, + } + } + }; + self.stop().await + } + + /// Immediately stops the GRPC service and waits for it to finish. This function ignores the [`ShutdownConfig`]. + pub async fn stop(mut self) -> Result<()> { + let inner = self.inner.take().expect("invariant: missing inner"); + inner.shutdown_tx.send(()).ok(); + let _ = inner.join_handle.await; + Ok(()) + } +} + +impl Drop for GrpcHandle { + fn drop(&mut self) { + if let Some(inner) = self.inner.take() { + inner.shutdown_tx.send(()).ok(); + } + } +} diff --git a/osprey_coordinator/src/pigeon/option_pin.rs b/osprey_coordinator/src/pigeon/option_pin.rs new file mode 100644 index 0000000..e396b59 --- /dev/null +++ b/osprey_coordinator/src/pigeon/option_pin.rs @@ -0,0 +1,12 @@ +use pin_project::pin_project; + +/// A pin-project compatible `Option`. +/// Copy of [`tonic::util::OptionPin`] +#[pin_project(project = OptionPinProj)] +pub(crate) enum OptionPin { + Some { + #[pin] + inner: T, + }, + None, +} diff --git a/osprey_coordinator/src/pigeon/pigeon_metrics.rs b/osprey_coordinator/src/pigeon/pigeon_metrics.rs new file mode 100644 index 0000000..d866bcf --- /dev/null +++ b/osprey_coordinator/src/pigeon/pigeon_metrics.rs @@ -0,0 +1,632 @@ +use crate::metrics::string_interner::{InternResult, StringInterner, Transformed}; +use crate::metrics::v2::{BaseMetric, DurationResolution}; +use crate::metrics::{MetricsEmitter, TagKey}; +use futures::ready; +use heck::SnakeCase; +use http::{HeaderMap, Request, Response}; +use http_body::Body; +use lazy_static::lazy_static; +use pin_project::pin_project; +use std::collections::{hash_map::Entry, HashMap}; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; +use std::time::Instant; +use std::{borrow::Cow, sync::atomic::AtomicUsize}; +use tonic::{Code, Status}; +use tower::{Layer, Service}; +use tracing::error; + +use super::utils::parse_path; + +pub(crate) const SERVICE_METADATA_KEY: &str = "service"; + +const ERROR_TYPE_CLIENT: &str = "client"; +const ERROR_TYPE_SERVER: &str = "server"; + +/// Metrics that are prefixed by the running service's name. +#[derive(Debug)] +pub(crate) struct LegacyPigeonMetrics { + server_rpc_response_time_name: String, + server_rpc_response_time: crate::metrics::v2::DurationHistogram, +} + +impl LegacyPigeonMetrics { + fn new(prefix: impl AsRef) -> Self { + LegacyPigeonMetrics { + server_rpc_response_time_name: format!("{}.rpc.response_time", prefix.as_ref()), + server_rpc_response_time: crate::metrics::v2::DurationHistogram::default(), + } + } +} + +impl crate::metrics::v2::MetricsEmitter for LegacyPigeonMetrics { + fn emit_metrics( + &self, + client: &crate::metrics::v2::macro_support::StatsdClient, + base_tags: &mut dyn Iterator, + ) -> Result<(), crate::metrics::v2::macro_support::MetricError> { + self.server_rpc_response_time.send_metric( + client, + &self.server_rpc_response_time_name, + base_tags, + )?; + Ok(()) + } +} + +#[derive(Clone, Debug, TagKey, PartialEq, Eq, Hash)] +struct LegacyServerRpcResponseTimeKey { + service: &'static str, + method: InternResult, + #[tag(name = "peer.service")] + peer_service: InternResult, +} + +#[derive(Debug, MetricsEmitter)] +pub(crate) struct PigeonMetrics { + #[metric(name = "pigeon.rpc.count")] + server_rpc_count: crate::metrics::v2::Counter>, + #[metric(name = "pigeon.rpc.error_count")] + server_rpc_error_count: crate::metrics::v2::Counter>, + #[metric(name = "pigeon.rpc.trailer_response_time")] + server_rpc_trailer_response_time: crate::metrics::v2::DurationHistogram>, + #[metric(name = "pigeon.rpc.response_time")] + server_rpc_response_time: crate::metrics::v2::DurationHistogram>, + /// The number of currently active server requests + #[metric(name = "pigeon.rpc.active")] + server_rpc_active: crate::metrics::v2::Gauge, f64>, + + #[metric(name = "pigeon.client_rpc.count")] + client_rpc_count: crate::metrics::v2::Counter>, + #[metric(name = "pigeon.client_rpc.error_count")] + client_rpc_error_count: crate::metrics::v2::Counter>, + #[metric(name = "pigeon.client_rpc.response_time")] + client_rpc_response_time: crate::metrics::v2::DurationHistogram>, + /// The number of currently active client requests + #[metric(name = "pigeon.rpc.active")] + client_rpc_active: crate::metrics::v2::Gauge, f64>, +} + +impl Default for PigeonMetrics { + fn default() -> Self { + PigeonMetrics { + server_rpc_count: Default::default(), + server_rpc_error_count: Default::default(), + server_rpc_response_time: crate::metrics::v2::DurationHistogram::new( + DurationResolution::Microseconds, + ), + server_rpc_trailer_response_time: crate::metrics::v2::DurationHistogram::new( + DurationResolution::Microseconds, + ), + server_rpc_active: Default::default(), + client_rpc_count: Default::default(), + client_rpc_error_count: Default::default(), + client_rpc_response_time: crate::metrics::v2::DurationHistogram::new( + DurationResolution::Microseconds, + ), + client_rpc_active: Default::default(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, TagKey)] +pub(crate) struct RpcErrorTags { + #[tag(name = "grpc.service")] + service: &'static str, + #[tag(name = "grpc.method")] + method: &'static str, + #[tag(name = "grpc.code")] + code: Code, + #[tag(name = "peer.service")] + peer_service: PeerService, + #[tag(name = "type")] + error_type: &'static str, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, TagKey)] +pub(crate) struct RpcTags { + #[tag(name = "grpc.service")] + service: &'static str, + #[tag(name = "grpc.method")] + method: &'static str, + #[tag(name = "peer.service")] + peer_service: PeerService, +} + +lazy_static! { + static ref METRICS: Arc = { + let m = Arc::new(PigeonMetrics::default()); + crate::metrics::v2::register_emitter(m.clone()).emit_forever(); + m + }; + static ref LEGACY_METRICS_MAP: Arc>>> = + Arc::new(Mutex::new(HashMap::new())); + static ref PEER_INTERNER: StringInterner = StringInterner::new(512); + static ref SERVICE_INTERNER: StringInterner = StringInterner::new(1024); + static ref METHOD_INTERNER: StringInterner = + StringInterner::with_transformer(1024, SnakeCase::to_snake_case); +} + +/// Gets or creates the metrics with the given prefix. +/// Will also register the metrics with the MetricsSender +/// the first time metrics with a particular prefix are requested. +pub(crate) fn get_legacy_metrics_and_ensure_registered<'a>( + prefix: impl Into>, +) -> anyhow::Result> { + let prefix = prefix.into(); + + let mut map = LEGACY_METRICS_MAP.lock().unwrap(); + match map.entry(prefix.into_owned()) { + Entry::Vacant(ent) => { + // First thread to create metrics registers it. + let pigeon_metrics = Arc::new(LegacyPigeonMetrics::new(ent.key())); + crate::metrics::v2::register_emitter(pigeon_metrics.clone()).emit_forever(); + ent.insert(pigeon_metrics.clone()); + Ok(pigeon_metrics) + } + // Another thread created the metrics first. + Entry::Occupied(ent) => Ok(ent.get().clone()), + } +} + +/// A [`tower::Layer`] that decorates a gRPC server or client channel +/// by instrumenting it with metrics. +#[derive(Clone, Debug)] +pub struct MetricsLayer { + v1_enabled: bool, + metrics: Arc, + role: MetricsServiceRole, + request_counter: Option>, +} + +impl MetricsLayer { + /// Creates a new server layer with the given metrics prefix. + /// This is typically the name of the running service (e.g. `"osprey_foo"`). + pub fn server<'a>(prefix: impl Into>) -> Self { + let prefix: &'static str = Box::leak(prefix.into().into_owned().into_boxed_str()); + MetricsLayer { + v1_enabled: true, + metrics: METRICS.clone(), + role: MetricsServiceRole::Server { + legacy_metrics: get_legacy_metrics_and_ensure_registered(prefix).unwrap(), + }, + request_counter: None, + } + } + + /// Creates a new client layer + /// with the name of the running service (e.g. `"osprey_foo"`). + pub fn client<'a>(peer_service: impl Into<&'static str>) -> Self { + MetricsLayer { + v1_enabled: true, + metrics: METRICS.clone(), + role: MetricsServiceRole::Client { + peer_service: peer_service.into(), + }, + request_counter: None, + } + } + + /// Stop emitting v1 prefixed metrics. + pub fn disable_metrics_v1(mut self) -> Self { + self.v1_enabled = false; + self + } + + /// Attaches a request counter to the metrics layer, which will be incremented each time + /// a request is served. + pub fn with_request_counter(self, request_counter: Arc) -> Self { + MetricsLayer { + request_counter: Some(request_counter), + ..self + } + } +} + +impl Layer for MetricsLayer { + type Service = MetricsService; + + fn layer(&self, inner: S) -> Self::Service { + MetricsService { + inner, + v1_enabled: self.v1_enabled, + metrics: self.metrics.clone(), + role: self.role.clone(), + request_counter: self.request_counter.clone(), + } + } +} + +/// Instruments an underlying service with metrics. +#[derive(Clone)] +pub struct MetricsService { + inner: S, + v1_enabled: bool, + metrics: Arc, + role: MetricsServiceRole, + request_counter: Option>, +} + +impl Service> for MetricsService +where + S: Service, Response = Response>, + S::Future: Send + 'static, + RespBody: Body, +{ + type Response = Response>; + type Error = S::Error; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, request: Request) -> Self::Future { + let start = Instant::now(); + let peer_service = request + .headers() + .get(SERVICE_METADATA_KEY) + .and_then(|v| v.to_str().ok()) + .map(|s| (*PEER_INTERNER).intern(s)) + .unwrap_or_else(|| InternResult::Interned("n/a")); + let parsed_path = parse_path(request.uri().path()).and_then(intern_path); + if let Some((service, method)) = parsed_path { + match self.role { + MetricsServiceRole::Server { .. } => { + self.metrics.server_rpc_count.incr(RpcTags { + service, + method, + peer_service, + }); + self.metrics.server_rpc_active.incr( + RpcTags { + service, + method, + peer_service, + }, + 1.0, + ); + } + MetricsServiceRole::Client { peer_service } => { + self.metrics.client_rpc_count.incr(RpcTags { + service, + method, + peer_service, + }); + self.metrics.client_rpc_active.incr( + RpcTags { + service, + method, + peer_service, + }, + 1.0, + ); + } + } + } + let metrics = self.metrics.clone(); + if let Some(request_counter) = &self.request_counter { + request_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + + let future = self.inner.call(request); + let role = self.role.clone(); + let v1_enabled = self.v1_enabled; + Box::pin(async move { + let result = future.await; + if let Some((service, method)) = parsed_path { + // When the future completes, that indicates the server handler has finished. + match role { + // On server-side, we want to record this as the response time, + // since the subsequent body will be dependent on the client and network conditions. + MetricsServiceRole::Server { .. } => { + metrics.server_rpc_response_time.record( + RpcTags { + service, + method, + peer_service, + }, + Instant::now().duration_since(start), + ); + metrics.server_rpc_active.decr( + RpcTags { + service, + method, + peer_service, + }, + 1.0, + ); + } + // On the client-side, we're not terribly interested in this timing. + // We won't return control flow to the application + // until we receive the trailers with the status, + // so the end of trailers may as well be the total response time. + // We will decrement the active gauge, though. + MetricsServiceRole::Client { peer_service } => { + metrics.client_rpc_active.decr( + RpcTags { + service, + method, + peer_service, + }, + 1.0, + ); + } + } + } + + match result { + Ok(response) => { + let header_status_code = Status::from_header_map(response.headers()) + .as_ref() + .map(Status::code); + // gRPC status for responses with a body is in the Trailers, + // so we wrap the inner service's returned body + // to increment the metrics after the trailers have been read. + Ok(response.map(|inner| MetricsServiceBody { + inner, + state: MetricsBodyState { + header_status_code, + pending_metrics: parsed_path.map(|(service, method)| PendingMetrics { + v1_enabled, + start, + service, + method, + peer_service, + metrics, + role, + }), + }, + })) + } + Err(err) => { + if let Some((service, method)) = parsed_path { + match role { + MetricsServiceRole::Server { .. } => { + metrics.server_rpc_error_count.incr(RpcErrorTags { + service, + method, + code: Code::Internal, + error_type: ERROR_TYPE_SERVER, + peer_service, + }); + } + MetricsServiceRole::Client { peer_service } => { + // Network error prevented reading the headers. + // gRPC canonically treats this as UNAVAILABLE. + metrics.client_rpc_error_count.incr(RpcErrorTags { + service, + method, + code: Code::Unavailable, + error_type: ERROR_TYPE_SERVER, + peer_service, + }); + } + } + } + Err(err) + } + } + }) + } +} + +fn intern_path((service, method): (&str, &str)) -> Option<(&'static str, &'static str)> { + let service = match (*SERVICE_INTERNER).intern(service) { + InternResult::Interned(s) => s, + InternResult::AtCapacity => { + error!( + "Could not add metric for /{}/{}: service intern table at capacity", + service, method, + ); + return None; + } + }; + let method = match (*METHOD_INTERNER).intern(method) { + InternResult::Interned(s) => s, + InternResult::AtCapacity => { + error!( + "Could not add metric for /{}/{}: method intern table at capacity", + service, method, + ); + return None; + } + }; + Some((service, method)) +} + +#[derive(Debug)] +struct MetricsBodyState { + header_status_code: Option, + pending_metrics: Option, +} + +impl MetricsBodyState { + fn record(&mut self, status_code: Code, end: Instant) { + if let Some(metrics) = self.pending_metrics.take() { + metrics.record(status_code, end); + } + } +} + +impl Drop for MetricsBodyState { + /// Ensures that the metrics are recorded, + /// using the time that the state is dropped as the RPC end time. + /// + /// gRPC error responses do not contain a body + /// so Tonic never bothers to poll the body. + /// Implementing `Drop` guarantees that the metrics will be recorded. + fn drop(&mut self) { + if let Some(metrics) = self.pending_metrics.take() { + let code = self.header_status_code.unwrap_or(Code::Unknown); + metrics.record(code, Instant::now()); + } + } +} + +#[derive(Clone, Debug)] +enum MetricsServiceRole { + Server { + legacy_metrics: Arc, + }, + Client { + peer_service: &'static str, + }, +} + +#[derive(Debug)] +struct PendingMetrics { + start: Instant, + service: &'static str, + method: &'static str, + peer_service: InternResult, + v1_enabled: bool, + metrics: Arc, + role: MetricsServiceRole, +} + +impl PendingMetrics { + fn record(self, status_code: Code, trailer_end: Instant) { + match &self.role { + // Disable legacy metrics if asked + MetricsServiceRole::Server { .. } if !self.v1_enabled => {} + MetricsServiceRole::Server { legacy_metrics } => { + let trailer_response_time = trailer_end.duration_since(self.start); + + legacy_metrics.server_rpc_response_time.record( + LegacyServerRpcResponseTimeKey { + service: self.service, + method: (*METHOD_INTERNER).intern(self.method), + peer_service: self.peer_service, + }, + trailer_response_time, + ); + + self.metrics.server_rpc_trailer_response_time.record( + RpcTags { + service: self.service, + method: self.method, + peer_service: self.peer_service, + }, + trailer_response_time, + ); + } + MetricsServiceRole::Client { peer_service } => { + self.metrics.client_rpc_response_time.record( + RpcTags { + service: self.service, + method: self.method, + peer_service, + }, + trailer_end.duration_since(self.start), + ); + } + } + match status_code { + Code::Ok => {} + Code::Cancelled + | Code::InvalidArgument + | Code::NotFound + | Code::AlreadyExists + | Code::PermissionDenied + | Code::FailedPrecondition + | Code::Aborted + | Code::OutOfRange + | Code::Unauthenticated => match self.role { + MetricsServiceRole::Server { .. } => { + self.metrics.server_rpc_error_count.incr(RpcErrorTags { + service: self.service, + method: self.method, + code: status_code, + error_type: ERROR_TYPE_CLIENT, + peer_service: self.peer_service, + }); + } + MetricsServiceRole::Client { peer_service } => { + self.metrics.client_rpc_error_count.incr(RpcErrorTags { + service: self.service, + method: self.method, + code: status_code, + error_type: ERROR_TYPE_CLIENT, + peer_service, + }); + } + }, + _ => match &self.role { + MetricsServiceRole::Server { .. } => { + self.metrics.server_rpc_error_count.incr(RpcErrorTags { + service: self.service, + method: self.method, + code: status_code, + error_type: ERROR_TYPE_SERVER, + peer_service: self.peer_service, + }); + } + MetricsServiceRole::Client { peer_service } => { + self.metrics.client_rpc_error_count.incr(RpcErrorTags { + service: self.service, + method: self.method, + code: status_code, + error_type: ERROR_TYPE_SERVER, + peer_service, + }); + } + }, + } + } +} + +/// Body of a [`ClientService`] response. +#[pin_project] +pub struct MetricsServiceBody { + #[pin] + inner: B, + state: MetricsBodyState, +} + +impl Body for MetricsServiceBody { + type Data = B::Data; + type Error = B::Error; + + fn poll_data( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll>> { + let this = self.project(); + let result = ready!(this.inner.poll_data(cx)); + if let Some(Err(_)) = &result { + this.state.record(Code::Unavailable, Instant::now()); + } + Poll::Ready(result) + } + + fn poll_trailers( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>> { + let this = self.project(); + let result = ready!(this.inner.poll_trailers(cx)); + let end = Instant::now(); + let status_code = match &result { + Ok(Some(trailer)) => Status::from_header_map(trailer) + .as_ref() + .map(Status::code) + .or(this.state.header_status_code), + Ok(None) => this.state.header_status_code, + Err(_) => Some(Code::Unavailable), + } + .unwrap_or(Code::Unknown); + this.state.record(status_code, end); + Poll::Ready(result) + } + + fn is_end_stream(&self) -> bool { + self.inner.is_end_stream() + } + + fn size_hint(&self) -> http_body::SizeHint { + self.inner.size_hint() + } +} diff --git a/osprey_coordinator/src/pigeon/recover_error.rs b/osprey_coordinator/src/pigeon/recover_error.rs new file mode 100644 index 0000000..6c52835 --- /dev/null +++ b/osprey_coordinator/src/pigeon/recover_error.rs @@ -0,0 +1,137 @@ +//! Copy of +//! https://github.com/hyperium/tonic/blob/v0.6.2/tonic/src/transport/server/recover_error.rs + +use futures::ready; +use http::Response; +use pin_project::pin_project; +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll}, +}; +use tower::Service; + +use super::option_pin::{OptionPin, OptionPinProj}; +use super::status::{add_status_header, try_status_from_error}; + +/// Middleware that attempts to recover from service errors by turning them into a response built +/// from the `Status`. +#[derive(Debug, Clone)] +pub(crate) struct RecoverError { + inner: S, +} + +impl RecoverError { + pub(crate) fn new(inner: S) -> Self { + Self { inner } + } +} + +impl Service for RecoverError +where + S: Service>, + S::Error: Into, +{ + type Response = Response>; + type Error = super::utils::Error; + type Future = ResponseFuture; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(Into::into) + } + + fn call(&mut self, req: R) -> Self::Future { + ResponseFuture { + inner: self.inner.call(req), + } + } +} + +#[pin_project] +pub(crate) struct ResponseFuture { + #[pin] + inner: F, +} + +impl Future for ResponseFuture +where + F: Future, E>>, + E: Into, +{ + type Output = Result>, super::utils::Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let result: Result, super::utils::Error> = + ready!(self.project().inner.poll(cx)).map_err(Into::into); + + match result { + Ok(response) => { + let response = response.map(MaybeEmptyBody::full); + Poll::Ready(Ok(response)) + } + Err(err) => match try_status_from_error(err) { + Ok(status) => { + let mut res = Response::new(MaybeEmptyBody::empty()); + add_status_header(&status, res.headers_mut()).unwrap(); + Poll::Ready(Ok(res)) + } + Err(err) => Poll::Ready(Err(err)), + }, + } + } +} + +#[pin_project] +pub(crate) struct MaybeEmptyBody { + #[pin] + inner: OptionPin, +} + +impl MaybeEmptyBody { + fn full(inner: B) -> Self { + Self { + inner: OptionPin::Some { inner }, + } + } + + fn empty() -> Self { + Self { + inner: OptionPin::None, + } + } +} + +impl http_body::Body for MaybeEmptyBody +where + B: http_body::Body + Send, +{ + type Data = B::Data; + type Error = B::Error; + + fn poll_data( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll>> { + match self.project().inner.project() { + OptionPinProj::Some { inner: b } => b.poll_data(cx), + OptionPinProj::None => Poll::Ready(None), + } + } + + fn poll_trailers( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>> { + match self.project().inner.project() { + OptionPinProj::Some { inner: b } => b.poll_trailers(cx), + OptionPinProj::None => Poll::Ready(Ok(None)), + } + } + + fn is_end_stream(&self) -> bool { + match &self.inner { + OptionPin::Some { inner: b } => b.is_end_stream(), + OptionPin::None => true, + } + } +} diff --git a/osprey_coordinator/src/pigeon/router.rs b/osprey_coordinator/src/pigeon/router.rs new file mode 100644 index 0000000..ebec555 --- /dev/null +++ b/osprey_coordinator/src/pigeon/router.rs @@ -0,0 +1,68 @@ +use super::status::GRPC_STATUS_HEADER_CODE; +use super::utils::{parse_path, Error}; +use futures::{future::BoxFuture, TryFutureExt}; +use http::{Request, Response, StatusCode}; +use hyper::Body; +use std::collections::HashMap; +use std::fmt::{self, Debug, Formatter}; +use std::future; +use tonic::body::BoxBody; +use tower::Service as TowerService; + +/// Dispatcher for the [`Server`](super::Server). +#[derive(Clone)] +pub(crate) struct Router { + pub(crate) services: HashMap<&'static str, S>, +} + +impl TowerService> for Router +where + S: TowerService, Response = Response> + Clone + Send + 'static, + S::Error: Into, + S::Future: Send + 'static, +{ + type Response = Response; + type Error = Error; + type Future = BoxFuture<'static, std::result::Result, Error>>; + + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn call(&mut self, req: Request) -> Self::Future { + let maybe_service = + parse_path(req.uri().path()).and_then(|(name, _)| self.services.get(name)); + match maybe_service { + // TODO: Technically, this should call poll_ready before calling the service, + Some(service) => Box::pin(service.clone().call(req).map_err(Into::into)), + None => Box::pin(future::ready(Ok(unimplemented_response()))), + } + } +} + +impl Debug for Router { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str("Router { services: {")?; + let mut keys: Vec<_> = self.services.keys().copied().collect(); + keys.sort_unstable(); + for (i, key) in keys.into_iter().enumerate() { + write!(f, "{} {}: Service", if i == 0 { "" } else { "," }, key)?; + } + f.write_str("} }")?; + Ok(()) + } +} + +fn unimplemented_response() -> Response { + // Copied from tonic's Unimplemented + // See https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#responses + http::Response::builder() + .status(StatusCode::OK) + .header(GRPC_STATUS_HEADER_CODE, tonic::Code::Unimplemented as i32) + .header("content-type", "application/grpc") + .body(tonic::body::empty_body()) + .unwrap() +} diff --git a/osprey_coordinator/src/pigeon/status.rs b/osprey_coordinator/src/pigeon/status.rs new file mode 100644 index 0000000..7c95673 --- /dev/null +++ b/osprey_coordinator/src/pigeon/status.rs @@ -0,0 +1,187 @@ +//! Copy of private methods from +//! https://github.com/hyperium/tonic/blob/v0.6.2/tonic/src/status.rs + +use bytes::Bytes; +use http::{HeaderMap, HeaderValue}; +use percent_encoding::{percent_encode, AsciiSet, CONTROLS}; +use std::{borrow::Cow, error::Error, fmt}; +use tonic::{metadata::MetadataMap, Code, Status}; +use tracing::debug; + +use super::grpc_timeout::TimeoutExpired; + +const ENCODING_SET: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'"') + .add(b'#') + .add(b'<') + .add(b'>') + .add(b'`') + .add(b'?') + .add(b'{') + .add(b'}'); + +pub(crate) const GRPC_STATUS_HEADER_CODE: &str = "grpc-status"; +const GRPC_STATUS_MESSAGE_HEADER: &str = "grpc-message"; +const GRPC_STATUS_DETAILS_HEADER: &str = "grpc-status-details-bin"; + +pub(crate) fn try_status_from_error( + err: Box, +) -> Result> { + let err = match err.downcast::() { + Ok(status) => { + return Ok(*status); + } + Err(err) => err, + }; + + let err = match err.downcast::() { + Ok(h2) => { + return Ok(from_h2_error(&*h2)); + } + Err(err) => err, + }; + + if let Some(status) = find_status_in_source_chain(&*err) { + // Difference from upstream: + // can't set source because it is private to tonic. + return Ok(status); + } + + Err(err) +} + +fn find_status_in_source_chain(err: &(dyn Error + 'static)) -> Option { + let mut source = Some(err); + + while let Some(err) = source { + if let Some(status) = err.downcast_ref::() { + return Some(Status::with_details_and_metadata( + status.code(), + status.message().to_string(), + Bytes::copy_from_slice(status.details()), + status.metadata().clone(), + )); + } + + if let Some(timeout) = err.downcast_ref::() { + return Some(Status::cancelled(timeout.to_string())); + } + + if let Some(hyper) = err + .downcast_ref::() + .and_then(from_hyper_error) + { + return Some(hyper); + } + + source = err.source(); + } + + None +} + +fn from_h2_error(err: &h2::Error) -> Status { + // See https://github.com/grpc/grpc/blob/3977c30/doc/PROTOCOL-HTTP2.md#errors + let code = match err.reason() { + Some(h2::Reason::NO_ERROR) + | Some(h2::Reason::PROTOCOL_ERROR) + | Some(h2::Reason::INTERNAL_ERROR) + | Some(h2::Reason::FLOW_CONTROL_ERROR) + | Some(h2::Reason::SETTINGS_TIMEOUT) + | Some(h2::Reason::COMPRESSION_ERROR) + | Some(h2::Reason::CONNECT_ERROR) => Code::Internal, + Some(h2::Reason::REFUSED_STREAM) => Code::Unavailable, + Some(h2::Reason::CANCEL) => Code::Cancelled, + Some(h2::Reason::ENHANCE_YOUR_CALM) => Code::ResourceExhausted, + Some(h2::Reason::INADEQUATE_SECURITY) => Code::PermissionDenied, + + _ => Code::Unknown, + }; + + // Difference from upstream: + // can't set source because it is private to tonic. + Status::new(code, format!("h2 protocol error: {}", err)) +} + +fn from_hyper_error(err: &hyper::Error) -> Option { + if err.is_timeout() || err.is_connect() { + return Some(Status::unavailable(err.to_string())); + } + None +} + +pub(crate) fn add_status_header(status: &Status, header_map: &mut HeaderMap) -> Result<(), Status> { + header_map.extend(metadata_into_sanitized_headers(status.metadata().clone())); + + header_map.insert(GRPC_STATUS_HEADER_CODE, code_to_header_value(status.code())); + + if !status.message().is_empty() { + let to_write = Bytes::copy_from_slice( + Cow::from(percent_encode(status.message().as_bytes(), ENCODING_SET)).as_bytes(), + ); + + header_map.insert( + GRPC_STATUS_MESSAGE_HEADER, + HeaderValue::from_maybe_shared(to_write).map_err(invalid_header_value_byte)?, + ); + } + + if !status.details().is_empty() { + use base64::engine::{general_purpose::STANDARD_NO_PAD, Engine}; + let details = STANDARD_NO_PAD.encode(status.details()); + + header_map.insert( + GRPC_STATUS_DETAILS_HEADER, + HeaderValue::from_maybe_shared(details).map_err(invalid_header_value_byte)?, + ); + } + + Ok(()) +} + +fn code_to_header_value(code: Code) -> HeaderValue { + match code { + Code::Ok => HeaderValue::from_static("0"), + Code::Cancelled => HeaderValue::from_static("1"), + Code::Unknown => HeaderValue::from_static("2"), + Code::InvalidArgument => HeaderValue::from_static("3"), + Code::DeadlineExceeded => HeaderValue::from_static("4"), + Code::NotFound => HeaderValue::from_static("5"), + Code::AlreadyExists => HeaderValue::from_static("6"), + Code::PermissionDenied => HeaderValue::from_static("7"), + Code::ResourceExhausted => HeaderValue::from_static("8"), + Code::FailedPrecondition => HeaderValue::from_static("9"), + Code::Aborted => HeaderValue::from_static("10"), + Code::OutOfRange => HeaderValue::from_static("11"), + Code::Unimplemented => HeaderValue::from_static("12"), + Code::Internal => HeaderValue::from_static("13"), + Code::Unavailable => HeaderValue::from_static("14"), + Code::DataLoss => HeaderValue::from_static("15"), + Code::Unauthenticated => HeaderValue::from_static("16"), + } +} + +const GRPC_RESERVED_HEADERS: [&str; 6] = [ + "te", + "user-agent", + "content-type", + "grpc-message", + "grpc-message-type", + "grpc-status", +]; + +fn metadata_into_sanitized_headers(mut metadata: MetadataMap) -> http::HeaderMap { + for r in &GRPC_RESERVED_HEADERS { + metadata.remove(*r); + } + metadata.into_headers() +} + +fn invalid_header_value_byte(err: Error) -> Status { + debug!("Invalid header: {}", err); + Status::new( + Code::Internal, + "Couldn't serialize non-text grpc status header".to_string(), + ) +} diff --git a/osprey_coordinator/src/pigeon/utils.rs b/osprey_coordinator/src/pigeon/utils.rs new file mode 100644 index 0000000..64d94b4 --- /dev/null +++ b/osprey_coordinator/src/pigeon/utils.rs @@ -0,0 +1,49 @@ +use tonic::transport::{Endpoint, Error as TonicError}; + +pub(crate) type Error = Box; + +/// Splits off the first component of a URL path. +/// As per the [gRPC spec], this corresponds to the gRPC service name and method name. +/// +/// [gRPC spec]: https://github.com/grpc/grpc/blob/99a7f2995e66f9d6f5c3403dc7178238251fb2b5/doc/PROTOCOL-HTTP2.md#requests +pub(crate) fn parse_path(path: &str) -> Option<(&str, &str)> { + let tail = match path.strip_prefix('/') { + Some(tail) => tail, + None => return None, + }; + let (service, method) = match tail.split_once('/') { + Some(split) => split, + None => return None, + }; + if service.is_empty() || method.is_empty() { + None + } else { + Some((service, method)) + } +} + +#[macro_export] +macro_rules! round_robin { + ( $client:expr, $method:ident, $request:expr ) => { + $client.round_robin_unary_unary($request, |mut c, r| async move { c.$method(r).await }) + }; +} + +pub trait ChunkedRequest { + type Key: Clone + ToString; + + // FIXME: Move this to use a GAT once that's stabilized. + fn iter_keys<'a>(&'a self) -> Box + 'a>; + fn with_keys(&self, keys: Vec) -> Self; +} + +#[tonic::async_trait] +pub trait Connector: Sized + Send { + async fn connect(dst: Endpoint) -> Result; + fn connect_lazy(dst: Endpoint) -> Self; +} + +pub trait ScalarRequest { + type Key: Clone + ToString; + fn get_key(&self) -> &Self::Key; +} diff --git a/osprey_coordinator/src/priority_queue.rs b/osprey_coordinator/src/priority_queue.rs new file mode 100644 index 0000000..7103615 --- /dev/null +++ b/osprey_coordinator/src/priority_queue.rs @@ -0,0 +1,250 @@ +use crate::metrics::gauges::StaticGauge; +use crate::metrics::histograms::StaticHistogram; +use tokio::{ + sync::oneshot, + time::{interval, Duration, Instant, MissedTickBehavior}, +}; + +use crate::{coordinator_metrics::SmiteCoordinatorMetrics, proto}; + +use crate::tokio_utils::AbortOnDrop; +use std::{cell::Cell, sync::Arc}; + +#[derive(Debug)] +pub enum AckOrNack { + Ack(Option), + Nack, +} + +impl From for AckOrNack { + fn from(ack_or_nack: proto::ack_or_nack::AckOrNack) -> Self { + match ack_or_nack { + proto::ack_or_nack::AckOrNack::Ack(inner) => Self::Ack(inner.verdicts), + proto::ack_or_nack::AckOrNack::Nack(_) => Self::Nack, + } + } +} + +pub struct AckableAction { + pub action: proto::SmiteCoordinatorAction, + acking_oneshot_sender: oneshot::Sender, + local_retry_count: Cell, + pub created_at: Instant, +} + +impl AckableAction { + pub fn new( + action: proto::SmiteCoordinatorAction, + ) -> ( + AckableAction, + oneshot::Receiver, + ) { + let (acking_oneshot_sender, acking_oneshot_receiver) = oneshot::channel::(); + let ackable_action = AckableAction { + action, + acking_oneshot_sender, + local_retry_count: 0.into(), + created_at: Instant::now(), + }; + (ackable_action, acking_oneshot_receiver) + } + + pub fn into_action(self) -> (proto::SmiteCoordinatorAction, ActionAcker) { + ( + self.action, + ActionAcker { + acking_oneshot_sender: self.acking_oneshot_sender, + }, + ) + } + + fn increment_retry_count(&self) { + let count = self.local_retry_count.get(); + self.local_retry_count.set(count + 1); + } + + #[allow(unused)] + pub fn retry_count(&self) -> u32 { + self.local_retry_count.get() + } +} + +#[derive(Debug)] +pub struct ActionAcker { + acking_oneshot_sender: oneshot::Sender, +} + +impl ActionAcker { + pub fn ack_or_nack>(self, ack_or_nack: T) { + self.acking_oneshot_sender.send(ack_or_nack.into()).ok(); + } +} + +pub enum Priority { + Sync, + Async, +} + +#[derive(Clone)] +pub struct PriorityQueueSender { + sync_sender: async_channel::Sender, + async_sender: async_channel::Sender, +} + +impl PriorityQueueSender { + fn new( + sync_sender: async_channel::Sender, + async_sender: async_channel::Sender, + ) -> PriorityQueueSender { + PriorityQueueSender { + sync_sender, + async_sender, + } + } + + pub fn close(&self) { + self.sync_sender.close(); + self.async_sender.close(); + } + pub async fn send_sync( + &self, + ackable_action: AckableAction, + ) -> Result<(), async_channel::SendError> { + self.send(ackable_action, Priority::Sync).await + } + + pub async fn send_async( + &self, + ackable_action: AckableAction, + ) -> Result<(), async_channel::SendError> { + self.send(ackable_action, Priority::Async).await + } + + async fn send( + &self, + ackable_action: AckableAction, + priority: Priority, + ) -> Result<(), async_channel::SendError> { + ackable_action.increment_retry_count(); + match priority { + Priority::Sync => self.sync_sender.send(ackable_action).await, + Priority::Async => self.async_sender.send(ackable_action).await, + } + } + + pub fn len_sync(&self) -> usize { + self.sync_sender.len() + } + + pub fn len_async(&self) -> usize { + self.async_sender.len() + } + + pub fn receiver_count_sync(&self) -> usize { + self.sync_sender.receiver_count() + } + + pub fn receiver_count_async(&self) -> usize { + self.async_sender.receiver_count() + } +} + +#[derive(Clone)] +pub struct PriorityQueueReceiver { + sync_receiver: async_channel::Receiver, + async_receiver: async_channel::Receiver, +} + +impl PriorityQueueReceiver { + fn new( + sync_receiver: async_channel::Receiver, + async_receiver: async_channel::Receiver, + ) -> PriorityQueueReceiver { + PriorityQueueReceiver { + sync_receiver, + async_receiver, + } + } + pub async fn recv( + &self, + metrics: Arc, + ) -> Result { + loop { + let result = tokio::select! { + biased; + result = self.sync_receiver.recv() => result, + result = self.async_receiver.recv() => match result { + Ok(ackable_action) => { + metrics.action_time_in_async_queue.record(Instant::now().duration_since(ackable_action.created_at)); + Ok(ackable_action) + } + Err(_) => self.sync_receiver.recv().await + }, + }; + match result { + Ok(ackable_action) => { + // If the acking oneshot receiver is closed then there is no reason to process this action + // This can happen if the client sending a sync classification request times out + if ackable_action.acking_oneshot_sender.is_closed() { + continue; + } else { + return Ok(ackable_action); + } + } + Err(err) => return Err(err), + } + } + } + + pub fn nack_all_async(&self) { + loop { + match self.async_receiver.try_recv() { + Ok(action) => match action.acking_oneshot_sender.send(AckOrNack::Nack) { + Ok(_) => (), + Err(_) => println!( + "tried to nack {:?} and the nacking receiver was dropped", + action.action + ), + }, + Err(_) => return, + } + } + } +} + +pub fn create_ackable_action_priority_queue() -> (PriorityQueueSender, PriorityQueueReceiver) { + let (sync_sender, sync_receiver) = async_channel::unbounded(); + let (async_sender, async_receiver) = async_channel::unbounded(); + ( + PriorityQueueSender::new(sync_sender, async_sender), + PriorityQueueReceiver::new(sync_receiver, async_receiver), + ) +} + +pub fn spawn_priority_queue_metrics_worker( + queue_sender: PriorityQueueSender, + metrics: Arc, +) -> AbortOnDrop<()> { + let mut interval = interval(Duration::from_millis(100)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + let join_handle = tokio::task::spawn(async move { + loop { + interval.tick().await; + metrics + .priority_queue_size_sync + .set(queue_sender.len_sync() as u64); + metrics + .priority_queue_size_async + .set(queue_sender.len_async() as u64); + metrics + .priority_queue_receiver_count_async + .set(queue_sender.receiver_count_async() as u64); + metrics + .priority_queue_receiver_count_sync + .set(queue_sender.receiver_count_sync() as u64); + } + }); + + AbortOnDrop::new(join_handle) +} diff --git a/osprey_coordinator/src/proto/mod.rs b/osprey_coordinator/src/proto/mod.rs new file mode 100644 index 0000000..bf74925 --- /dev/null +++ b/osprey_coordinator/src/proto/mod.rs @@ -0,0 +1,66 @@ +#![allow(unused_imports)] + +pub mod discord_smite_rpc { + pub mod actions { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/discord_smite_rpc.actions.v1.rs")); + } + } + + pub mod common { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/discord_smite_rpc.common.v1.rs")); + } + } + + pub mod labels { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/discord_smite_rpc.labels.v1.rs")); + } + } + + pub mod smite_coordinator { + pub mod bidirectional_stream { + pub mod v1 { + include!(concat!( + env!("OUT_DIR"), + "/discord_smite_rpc.smite_coordinator.bidirectional_stream.v1.rs" + )); + } + } + + pub mod sync_action { + pub mod v1 { + include!(concat!( + env!("OUT_DIR"), + "/discord_smite_rpc.smite_coordinator.sync_action.v1.rs" + )); + } + } + } +} + +pub mod discord_protos { + pub mod discord_authentication { + pub mod v1 { + include!(concat!( + env!("OUT_DIR"), + "/discord_protos.discord_authentication.v1.rs" + )); + } + } + + pub mod users { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/discord_protos.users.v1.rs")); + } + } +} + +pub use discord_smite_rpc::actions::v1::*; +pub use discord_smite_rpc::common::v1::*; +pub use discord_smite_rpc::labels::v1::*; +pub use discord_smite_rpc::smite_coordinator::bidirectional_stream::v1::*; +pub use discord_smite_rpc::smite_coordinator::sync_action::v1 as smite_coordinator_sync_action; + +pub const PB_DESCRIPTOR_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/descriptor.bin")); diff --git a/osprey_coordinator/src/pub_sub_streaming_pull/ack_id.rs b/osprey_coordinator/src/pub_sub_streaming_pull/ack_id.rs new file mode 100644 index 0000000..bbd8508 --- /dev/null +++ b/osprey_coordinator/src/pub_sub_streaming_pull/ack_id.rs @@ -0,0 +1,17 @@ +/// An AckId is basically an opaque type that is used to internally represent theAckId. +#[derive(Debug, Eq, PartialEq, Hash, Copy, Clone)] +pub(crate) struct AckId(u64); + +impl AckId { + /// Creates a new AckId, starting at the beginning of the sequence (0). + pub(crate) fn new() -> Self { + AckId(0) + } + + /// Returns the "next" AckId. In the event that the internal sequence overflows, it simply wraps around. + /// + /// Practically, the wrap-around of a u64 is impossible. + pub(crate) fn next(&self) -> AckId { + Self(self.0.wrapping_add(1)) + } +} diff --git a/osprey_coordinator/src/pub_sub_streaming_pull/flow_control.rs b/osprey_coordinator/src/pub_sub_streaming_pull/flow_control.rs new file mode 100644 index 0000000..7da0420 --- /dev/null +++ b/osprey_coordinator/src/pub_sub_streaming_pull/flow_control.rs @@ -0,0 +1,96 @@ +/// The minimum and default value for [`FlowControl::min_duration_per_please_extension`] +const MIN_LEASE_EXTENSION_DURATION_SECS: u32 = 10; + +/// The maximum and default value for [`FlowControl::max_duration_per_please_extension`] +const MAX_LEASE_EXTENSION_DURATION_SECS: u32 = 600; + +/// Flow control is used to control the buffering of messages for a given [`crate::StreamingPullManager`]. +#[derive(Clone, Debug)] +pub struct FlowControl { + pub(crate) max_bytes: usize, + pub(crate) max_messages: usize, + pub(crate) max_processing_messages: usize, + pub(crate) min_duration_per_lease_extension: u32, + pub(crate) max_duration_per_lease_extension: u32, +} + +impl Default for FlowControl { + fn default() -> Self { + Self::new() + } +} + +impl FlowControl { + pub const fn new() -> Self { + Self { + max_bytes: 100 * 1024 * 1024, + max_messages: 1000, + max_processing_messages: 1000, + min_duration_per_lease_extension: MIN_LEASE_EXTENSION_DURATION_SECS, + max_duration_per_lease_extension: MAX_LEASE_EXTENSION_DURATION_SECS, + } + } + + /// The maximum total size of receiveed - but not yet processed - messages before + /// pausing the message stream. + /// + /// Default: 100MiB. + pub const fn set_max_bytes(mut self, max_bytes: usize) -> Self { + self.max_bytes = max_bytes; + self + } + + /// The maximum number of messages that can be received without being acked/nacked before pausing the message + /// stream. + /// + /// Default: 1000 + pub const fn set_max_messages(mut self, max_messages: usize) -> Self { + self.max_messages = max_messages; + self + } + + /// The maximum number of messages that will be sent to the message handler, and be outstanding at a given time. + /// + /// This is essentially a limit to how many messages will be concurrently handled by the message handler. + /// + /// Default: 1000 + pub const fn set_max_processing_messages(mut self, max_processing_messages: usize) -> Self { + self.max_processing_messages = max_processing_messages; + self + } + + /// Sets minimum and maximum number of seconds that we will bump any lease forward. + /// + /// The minimum must be >= 10. + /// The maximum must be <= 600. + /// The maximum must be >= the minimum. + /// + /// Default: (min = 10 seconds, max = 600 seconds) + /// + /// # Panics + /// + /// Panics if any of the invariants above are not met. + pub fn set_duration_per_lease_extension(mut self, min: u32, max: u32) -> Self { + if min < MIN_LEASE_EXTENSION_DURATION_SECS { + panic!( + "Minimum of {} is under acceptable value of {}", + min, MIN_LEASE_EXTENSION_DURATION_SECS + ); + } + + if max > MAX_LEASE_EXTENSION_DURATION_SECS { + panic!( + "Maximum of {} is above acceptable value of {}", + max, MAX_LEASE_EXTENSION_DURATION_SECS + ); + } + + if min > max { + panic!("Minimum ({}) is greater than Maximum ({})", min, max); + } + + self.min_duration_per_lease_extension = min; + self.max_duration_per_lease_extension = max; + self + } +} diff --git a/osprey_coordinator/src/pub_sub_streaming_pull/message.rs b/osprey_coordinator/src/pub_sub_streaming_pull/message.rs new file mode 100644 index 0000000..46ed29d --- /dev/null +++ b/osprey_coordinator/src/pub_sub_streaming_pull/message.rs @@ -0,0 +1,266 @@ +use std::{borrow::Borrow, collections::HashMap, fmt, sync::Arc, time::Instant}; + +use prost_types::Timestamp; +use tokio::sync::mpsc::UnboundedSender; + +use crate::gcloud::google::pubsub::v1::PubsubMessage; + +use crate::pub_sub_streaming_pull::{AckId, AckOrNack, StreamingPullManagerMessage}; + +/// The message ack guard is used to ensure that when a message is dropped without being acked or nacked, it is nacked. +/// +/// Acknowledgement of a message is an explicit action that must be invoked via [`MessageAckGuard::ack`]. +/// +/// This is only exposed when you call [`Message::into_inner`]. +#[must_use] +pub struct MessageAckGuard { + state: MessageAckGuardState, +} + +enum MessageAckGuardState { + Unacked { + ack_id: AckId, + received_at: Instant, + sender: UnboundedSender, + }, + Acked, +} + +impl MessageAckGuard { + pub(crate) fn new( + sender: UnboundedSender, + received_at: Instant, + ack_id: AckId, + ) -> MessageAckGuard { + Self { + state: MessageAckGuardState::Unacked { + ack_id, + received_at, + sender, + }, + } + } + + /// See [`Message::ack`] for a description on what acking a message means. + pub fn ack(mut self) { + self.do_send(AckOrNack::Ack); + } + + /// See [`Message::nack`] for a description on what nacking a message means. + pub fn nack(mut self) { + self.do_send(AckOrNack::Nack); + } + + /// An internal method to mark that the message was acked elsewhere within the streaming pull manager. + pub(crate) fn mark_acked_elsewhere(mut self) -> AckId { + let ack_id = self.ack_id(); + self.state = MessageAckGuardState::Acked; + ack_id + } + + /// Sends an AckOrNack message to the streaming pull manager. + fn do_send(&mut self, f: impl FnOnce(AckId) -> AckOrNack) { + if let MessageAckGuardState::Unacked { + ack_id, + sender, + received_at, + } = std::mem::replace(&mut self.state, MessageAckGuardState::Acked) + { + sender + .send(StreamingPullManagerMessage::AckOrNack { + ack_or_nack: f(ack_id), + latency: received_at.elapsed(), + }) + .ok(); + } + } + + fn ack_id(&self) -> AckId { + match &self.state { + MessageAckGuardState::Unacked { ack_id, .. } => *ack_id, + MessageAckGuardState::Acked => unreachable!( + "invariant: ack_id should never be called after message has been ack or nacked." + ), + } + } +} + +impl Drop for MessageAckGuard { + fn drop(&mut self) { + self.do_send(AckOrNack::Nack); + } +} + +/// A wrapper class for a pub-sub message, which automatically will nack the message if dropped. +/// +/// In order to mark the message as processed, it is imperative to call [`Message::ack`], otherwise +/// the message will be re-delivered. +#[must_use] +pub struct Message { + inner_message: DetachedMessage, + guard: MessageAckGuard, +} + +impl fmt::Debug for Message { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Message") + .field("message_id", &self.message_id) + .finish() + } +} + +impl std::ops::Deref for Message { + type Target = DetachedMessage; + + fn deref(&self) -> &Self::Target { + &self.inner_message + } +} + +impl Message { + pub(crate) fn new(inner_message: DetachedMessage, guard: MessageAckGuard) -> Self { + Self { + inner_message, + guard, + } + } + + pub(crate) fn ack_id(&self) -> AckId { + self.guard.ack_id() + } + + /// Acknowledges the message, marking it as processed. After acknowledgement, a message should no longer be + /// delivered to another subscriber on this subscription. However, this is not a guarantee, as pub-sub is at-least + /// once delivery, not exactly-once. + pub fn ack(self) -> DetachedMessage { + self.guard.ack(); + self.inner_message + } + + /// Nacks the message, marking it as not processed. When a message is nacked, it will be re-delivered to another + /// subscriber. Often times immediately. + pub fn nack(self) -> DetachedMessage { + self.guard.nack(); + self.inner_message + } + + /// Converts this message into the [`DetachedMessage`] which will let you take ownership of the various fields within + /// the message. Also, returns the inner [`MessageAckGuard`] which you can use to ack/nack the message. + pub fn into_inner(self) -> (DetachedMessage, MessageAckGuard) { + (self.inner_message, self.guard) + } +} + +/// A message that has been detached from the [`crate::StreamingPullManager`] which sent it. +/// +/// Obtained using [`Message::into_inner`], this struct has no special drop behavior. +#[derive(Debug, Clone)] +pub struct DetachedMessage { + pub data: Vec, + pub ordering_key: Option>, + pub attributes: HashMap, + pub message_id: String, + delivery_attempt: u32, + publish_time: Timestamp, + size: usize, +} + +impl DetachedMessage { + pub(crate) fn new(m: PubsubMessage, delivery_attempt: u32) -> Self { + let size = m.data.len() + + m.message_id.len() + + m.ordering_key.len() + + m.attributes + .iter() + .map(|(k, v)| k.len() + v.len()) + .sum::(); + Self { + size, + data: m.data, + ordering_key: if m.ordering_key.is_empty() { + None + } else { + Some(m.ordering_key.into()) + }, + attributes: m.attributes, + message_id: m.message_id, + publish_time: m + .publish_time + .expect("invariant: message from pub-sub did not have a timestamp"), + delivery_attempt, + } + } + + /// The "size" of the message, as used for flow control purposes. This is NOT the length of the data field, but + /// sums up all the strings in the message to compute the size. + pub(crate) fn size(&self) -> usize { + self.size + } + + /// The message data field. If this field is empty, the message must contain at least one attribute. + pub fn data(&self) -> &[u8] { + &self.data + } + + /// Attempts to decode the data as a given [`prost::Message`]. + pub fn decode_data(&self) -> Result { + T::decode(self.data()) + } + + /// If non-empty, identifies related messages for which publish order should be respected. + /// If a Subscription has enable_message_ordering set to true, messages published with the same non-empty + /// ordering_key value will be delivered to subscribers in the order in which they are received by the Pub/Sub + /// system. All PubsubMessages published in a given PublishRequest must specify the same ordering_key value. + pub fn ordering_key(&self) -> Option<&str> { + self.ordering_key.as_deref() + } + + /// Attributes for this message. If this field is empty, the message must contain non-empty data. + pub fn attributes(&self) -> &HashMap { + &self.attributes + } + + /// Retrieves a given attribute. + pub fn get_attribute(&self, key: K) -> Option<&str> + where + String: Borrow, + K: std::hash::Hash + Eq, + { + self.attributes.get(&key).map(|x| x.as_ref()) + } + + /// ID of this message, assigned by the server when the message is published. Guaranteed to be unique within the + /// topic. + pub fn message_id(&self) -> &str { + &self.message_id + } + + /// The approximate number of times that Cloud Pub/Sub has attempted to deliver the associated message to a + /// subscriber. + /// + /// More precisely, this is 1 + (number of NACKs) + (number of ack_deadline exceeds) for this message. + /// + /// A NACK is any call to ModifyAckDeadline with a 0 deadline. An ack_deadline exceeds event is whenever a message + /// is not acknowledged within . + /// + /// Upon the first delivery of a given message, delivery_attempt will have a value of 1. The value is calculated at + /// best effort and is approximate. If a DeadLetterPolicy is not set on the subscription, this will be 0. + pub fn delivery_attempt(&self) -> u32 { + self.delivery_attempt + } + + /// The time at which the message received by the pub-sub server. + /// + /// It is important to note that the publish timestamp is generated when the server receives + /// the message. Thus, it is not a good representation of "end to end" latency of how long a + /// message takes to flow through through the system and be received by a subscriber, as + /// there is latency between when the message is enqueued for publishing on the publisher end, + /// and when that publish request is flushed and reaches the server (which is when the timestamp + /// is generated.) + /// + /// If you require knowing the end-to-end latency of when a message was created, and when it was + /// received, that must be stored out of bounds, either in the message data, or attributes. + pub fn publish_time(&self) -> Timestamp { + self.publish_time.clone() + } +} diff --git a/osprey_coordinator/src/pub_sub_streaming_pull/message_ack_queue.rs b/osprey_coordinator/src/pub_sub_streaming_pull/message_ack_queue.rs new file mode 100644 index 0000000..a5a72b6 --- /dev/null +++ b/osprey_coordinator/src/pub_sub_streaming_pull/message_ack_queue.rs @@ -0,0 +1,271 @@ +use std::{ + collections::VecDeque, + future::pending, + task::{Context, Poll}, + time::{Duration, Instant}, +}; + +use fxhash::FxHashMap; +use rand::{thread_rng, Rng}; +use tokio_util::time::{delay_queue::Key, DelayQueue}; + +use crate::pub_sub_streaming_pull::ack_id::AckId; + +pub(crate) enum AckOrNack { + Ack(T), + Nack(T), +} + +impl AckOrNack { + pub(crate) fn ack_id(&self) -> &T { + match self { + AckOrNack::Ack(ack_id) | AckOrNack::Nack(ack_id) => ack_id, + } + } + + pub(crate) fn map(self, f: F) -> AckOrNack + where + F: FnOnce(T) -> R, + { + match self { + AckOrNack::Ack(ack_id) => AckOrNack::Ack(f(ack_id)), + AckOrNack::Nack(ack_id) => AckOrNack::Nack(f(ack_id)), + } + } +} + +struct MessageAckQueueItem { + /// The ack id as provided by pub-sub (which is mapped in [`MessageAckQueue::ack_id_mapping`]). + ack_id_from_server: AckIdFromServerT, + /// The key for the lease renewal timer ([`MessageAckQueue::lease_renewal_delay_queue`]). + lease_delay_queue_key: Key, + /// Instant in which the message was enqueued. + enqueued_at: Instant, +} + +/// The message ack queue is responsible for taking in acks or nacks via `enqueue_ack_or_nack`, and then +/// combining those acks into a chunk that is then provided via `next_chunk`. +pub(crate) struct MessageAckQueue { + /// How big each chunk should be. If the number of enqueued acks exceeds this size, + /// `next_chunk` will produce chunks capped at this size, until the pending acks are + /// empty. + max_chunk_size: usize, + /// How long should we wait until our pending_acks queue fills up, till we flush an incomplete + /// chunk. + pending_ack_flush_interval: Duration, + /// When we should perform the next flush (meaning, we will yield a partial chunk.) + next_flush: Instant, + /// The array of pending acks (our queue). + current_chunk: Vec>, + /// Contains the "full" chunks, which have filled up to `max_chunk_size`. + full_chunks: VecDeque>>, + /// A mapping of a ack id => [`MessageAckQueueItem`]. + ack_id_mapping: FxHashMap>, + /// The last ack id we dispensed, so we can get the next ack id. + last_ack_id: AckId, + /// The lease renewal delay queue, which tracks when we should renew leases for a given item. + lease_renewal_delay_queue: DelayQueue, +} + +impl MessageAckQueue +where + AckIdFromServer: Clone, +{ + pub(crate) fn new( + flow_control_max_messages: usize, + max_chunk_size: usize, + pending_ack_flush_interval: Duration, + ) -> Self { + Self { + max_chunk_size, + pending_ack_flush_interval, + next_flush: Instant::now() + pending_ack_flush_interval, + current_chunk: Vec::with_capacity(max_chunk_size), + full_chunks: VecDeque::with_capacity(flow_control_max_messages / max_chunk_size), + ack_id_mapping: Default::default(), + last_ack_id: AckId::new(), + lease_renewal_delay_queue: DelayQueue::with_capacity(flow_control_max_messages), + } + } + + /// Returns the amount of pending acks/nacks that have yet to be flushed. + pub(crate) fn len(&self) -> usize { + self.current_chunk.len() + (self.full_chunks.len() * self.max_chunk_size) + } + + /// Returns True if there are any pending acks/nacks that have yet to be flushed. + pub(crate) fn is_empty(&self) -> bool { + self.current_chunk.is_empty() && self.full_chunks.is_empty() + } + + /// Collects the ack ids that need to be renewed when called. Will return at most `max_chunk_size` items to renew. + /// + /// The caller is expected to repeatedly call this function until it returns a chunk smaller than `max_chunk_size`, + /// which means that we have collected all expired lease items. + pub(crate) fn collect_ack_ids_that_need_to_be_renewed( + &mut self, + max_chunk_size: usize, + max_lease_renewal_delay: Duration, + ) -> Vec<(Duration, AckIdFromServer)> { + // Since we never care about waking up any future when the next item in the delay queue expires, we'll simply + // use a noop waker. + let noop_waker = futures::task::noop_waker(); + let mut context = Context::from_waker(&noop_waker); + + let now = tokio::time::Instant::now(); + let delay_range = (max_lease_renewal_delay / 2)..max_lease_renewal_delay; + let mut rng = thread_rng(); + + // Function that computes the next lease renewal instant, by spraying values between + // (max_lease_renewal_duration / 2)..max_lease_renewal_duration. + let mut next_lease_renewal_instant = || now + rng.gen_range(delay_range.clone()); + + let mut result = Vec::new(); + while result.len() < max_chunk_size { + let ack_id = match self.lease_renewal_delay_queue.poll_expired(&mut context) { + Poll::Ready(Some(ack_id)) => ack_id.into_inner(), + Poll::Ready(None) | Poll::Pending => break, + }; + + let ack_item = match self.ack_id_mapping.get_mut(&ack_id) { + Some(ack_item) => ack_item, + None => continue, + }; + + // Optimization: We have some expired items, so we'll go ahead and pre-reserve the entire chunk, to avoid + // re-allocations of the vector. + if result.capacity() == 0 { + result.reserve(max_chunk_size); + } + + // Re-schedule the lease renewal to the next deadline. + ack_item.lease_delay_queue_key = self + .lease_renewal_delay_queue + .insert_at(ack_id, next_lease_renewal_instant()); + + result.push(( + now.saturating_duration_since(ack_item.enqueued_at.into()), + ack_item.ack_id_from_server.clone(), + )); + } + + result + } + + /// Given an ack_id from the protobuf, exchange it for an [`AckId`], and store that we're currently holding onto + /// a message with the provided `ack_id_from_server`. + /// + /// We will relinquish our hold of the `ack_id_from_server` once the returned [`AckId`] is provided to + /// [`MessageAckQueue::enqueue_ack_or_nack`], and a batch containing the given AckId is collected by + /// [`MessageAckQueue::next_chunk`]. + pub(crate) fn transform_and_store_ack_id( + &mut self, + ack_id_from_server: AckIdFromServer, + received_at: Instant, + lease_renew_at: tokio::time::Instant, + ) -> AckId { + let next_ack_id = self.last_ack_id.next(); + self.last_ack_id = next_ack_id; + + let lease_delay_queue_key = self + .lease_renewal_delay_queue + .insert_at(next_ack_id, lease_renew_at); + + self.ack_id_mapping.insert( + next_ack_id, + MessageAckQueueItem { + ack_id_from_server, + lease_delay_queue_key, + enqueued_at: received_at, + }, + ); + + next_ack_id + } + + /// Enqueues an ack to the batch that will eventually be consumed by `take_batch`. This removes the + /// message from lease management. + pub(crate) fn enqueue_ack_or_nack(&mut self, ack_or_nack: AckOrNack) { + if let Some(ack_or_nack) = self.remove_ack_or_nack(ack_or_nack) { + // When the first message is placed into the ack queue, we'll mark the last flush as being + // the time in which the message was added to the queue, meaning it *should* linger in the queue + // for at most `pending_ack_flush_interval`. + if self.is_empty() { + self.reset_next_flush(); + } + + self.current_chunk.push(ack_or_nack); + + // If we have filled our chunk, we'll go ahead and move that chunk into the full chunk list, + // and allocate a new empty chunk. + if self.current_chunk.len() == self.max_chunk_size { + let chunk = self.take_current_chunk(); + self.full_chunks.push_back(chunk); + } + } + } + + /// Takes the current chunk, and replaces it with an empty chunk. + fn take_current_chunk(&mut self) -> Vec> { + std::mem::replace( + &mut self.current_chunk, + Vec::with_capacity(self.max_chunk_size), + ) + } + + /// Tries to take the next chunk of enqueued acks. + /// + /// This function has the following behavior: + /// - If there are no pending acks, this future will *never* resolve. + /// - If the number of pending acks is greater than or equal to `max_chunk_size`, this future will + /// resolve immediately with a Vec no larger than `max_chunk_size`, if there is additional remaining + /// we will not reset the last flush timestamp, such that the next call to this function will either resolve + /// instantly, or at the next flush interval. + /// - Otherwise, this future will resolve after after at least `pending_ack_flush_interval` with a non-empty + /// batch. + /// + /// NOTE: This future is cancel-safe. Cancelling this future before it is ready will not result in any lost chunks. + pub(crate) async fn next_chunk(&mut self) -> Vec> { + if self.is_empty() { + return pending().await; + } + + match self.full_chunks.pop_front() { + // We have a full chunk ready, so we can immediately dispense of the chunk. + Some(pending_ack_chunk) => pending_ack_chunk, + // We have no full chunk, so instead, we'll wait until `next_flush` before trying to dispense of the + // non-empty current chunk. + None => { + tokio::time::sleep_until(self.next_flush.into()).await; + self.take_current_chunk() + } + } + } + + /// Removes the [`AckId`] from an [`AckOrNack`] using `remove_ack_id`, and returns the ack_kd_from_server + /// as provided by [`MessageAckQueue::transform_and_store_ack_id`]. + fn remove_ack_or_nack(&mut self, ack_or_nack: AckOrNack) -> Option> { + self.remove_ack_id(ack_or_nack.ack_id()) + .map(|ack_id_from_server| ack_or_nack.map(|_| ack_id_from_server)) + } + + /// Removes an ack id from the `ack_id_mapping`, returning back the `ack_id_from_server` as provided by + /// [`MessageAckQueue::transform_and_store_ack_id`]. + pub(crate) fn remove_ack_id(&mut self, ack_id: &AckId) -> Option { + let MessageAckQueueItem { + ack_id_from_server, + lease_delay_queue_key, + enqueued_at: _, + } = self.ack_id_mapping.remove(ack_id)?; + + self.lease_renewal_delay_queue + .remove(&lease_delay_queue_key); + + Some(ack_id_from_server) + } + + /// Resets the next flush deadline by bumping it into the future by `pending_ack_flush_interval`. + fn reset_next_flush(&mut self) { + self.next_flush = Instant::now() + self.pending_ack_flush_interval; + } +} diff --git a/osprey_coordinator/src/pub_sub_streaming_pull/message_handler.rs b/osprey_coordinator/src/pub_sub_streaming_pull/message_handler.rs new file mode 100644 index 0000000..c5c48b8 --- /dev/null +++ b/osprey_coordinator/src/pub_sub_streaming_pull/message_handler.rs @@ -0,0 +1,64 @@ +use std::future::Future; + +use crate::tokio_utils::AbortOnDrop; + +use crate::pub_sub_streaming_pull::{DetachedMessage, Message}; + +/// The message handler is invoked by the streaming pull manager when there are messages to process. +/// +/// When the [`crate::StreamingPullManager`] is created, the message handler is moved into the streaming pull manager's +/// task. If the the message handler's `handle_messages` function panicks, this will cause the streaming pull manager to +/// panic as well. So, exercise caution to ensure this method does not panic (unless something really bad has happened). +pub trait MessageHandler: Send + Sync + 'static { + /// Invoked when there are more messages to handle. + /// + /// The streaming pull manager will limit the number of in-flight messages (that is to say, messages that are + /// outstanding that have not been acked/nacked) based upon the settings configured in [`crate::FlowControl`]. + /// + /// Thus, you should not need to implement any kind of concurrency limiting in your message handler, simply + /// configure the [`crate::FlowControl`] accordingly. + fn handle_messages(&mut self, messages: Vec); + + /// Invoked when the streaming pull manager begins its graceful stop process. After this method is invoked, + /// `handle_messages` will no longer be called. + /// + /// This method can return an optional AbortOnDrop, which will be awaited upon during the graceful stop procedure. + /// + /// Most implementers of `MessageHandler` will not need to implement this method, however, some may benefit + /// from the knowledge that there will be no further messages handled. + fn handle_graceful_stop(&mut self) -> Option { + None + } +} + +/// This utility struct will process each message by spawning a task that invokes the provided `message_handler`, +/// which will return a future that will then be driven to completion by [`tokio::task::spawn`]. The future must return +/// a [`Result<(), ()>`]. The message will be acked if the result is [`Ok`], or nacked if the result is [`Err`]. +pub struct SpawnTaskPerMessageHandler { + message_handler: F, +} + +impl SpawnTaskPerMessageHandler { + pub fn new(message_handler: F) -> Self { + Self { message_handler } + } +} + +impl MessageHandler for SpawnTaskPerMessageHandler +where + R: Future> + Send + 'static, + F: Fn(DetachedMessage) -> R + Send + Sync + 'static, +{ + fn handle_messages(&mut self, messages: Vec) { + for message in messages { + let (message, guard) = message.into_inner(); + let future = (self.message_handler)(message); + tokio::task::spawn(async move { + match future.await { + Ok(()) => guard.ack(), + Err(()) => guard.nack(), + } + }); + } + } +} diff --git a/osprey_coordinator/src/pub_sub_streaming_pull/messages_in_flight.rs b/osprey_coordinator/src/pub_sub_streaming_pull/messages_in_flight.rs new file mode 100644 index 0000000..82171ce --- /dev/null +++ b/osprey_coordinator/src/pub_sub_streaming_pull/messages_in_flight.rs @@ -0,0 +1,53 @@ +use std::sync::Arc; + +use fxhash::FxHashMap; + +use crate::pub_sub_streaming_pull::ack_id::AckId; + +struct MessageInFlight { + ordering_key: Option>, + size_in_bytes: usize, +} + +#[derive(Default)] +pub(crate) struct MessagesInFlight { + messages_in_flight: FxHashMap, + bytes_in_flight: usize, +} + +impl MessagesInFlight { + pub(crate) fn bytes_in_flight(&self) -> usize { + self.bytes_in_flight + } + + pub(crate) fn len(&self) -> usize { + self.messages_in_flight.len() + } + + pub(crate) fn is_empty(&self) -> bool { + self.messages_in_flight.is_empty() + } + + pub(crate) fn remove(&mut self, ack_id: &AckId) -> Option>> { + let removed = self.messages_in_flight.remove(ack_id)?; + self.bytes_in_flight -= removed.size_in_bytes; + Some(removed.ordering_key) + } + + pub(crate) fn insert(&mut self, message: &crate::pub_sub_streaming_pull::Message) { + let size_in_bytes = message.size(); + let message_in_flight = MessageInFlight { + ordering_key: message.ordering_key.clone(), + size_in_bytes, + }; + + if let Some(prev_in_flight) = self + .messages_in_flight + .insert(message.ack_id(), message_in_flight) + { + self.bytes_in_flight -= prev_in_flight.size_in_bytes; + } + + self.bytes_in_flight += size_in_bytes; + } +} diff --git a/osprey_coordinator/src/pub_sub_streaming_pull/messages_on_hold.rs b/osprey_coordinator/src/pub_sub_streaming_pull/messages_on_hold.rs new file mode 100644 index 0000000..53d8781 --- /dev/null +++ b/osprey_coordinator/src/pub_sub_streaming_pull/messages_on_hold.rs @@ -0,0 +1,141 @@ +use std::{collections::VecDeque, sync::Arc}; + +use fxhash::FxHashMap; + +use crate::pub_sub_streaming_pull::{ack_id::AckId, Message}; + +/// This struct exists for the sole purpose of reminding us to call `unwrap_message` when we remove the message +/// from `MessagesOnHold`. +struct WrappedMessage(Message); + +/// MessageOnHold is responsible for buffering messages sent from the pub-sub server, +/// holding onto the messages if there are too many in-flight, or if exact-ordering is +/// enabled, buffering messages within a given ordering_key, until a message for the +/// ordering key has completed. +#[derive(Default)] +pub(crate) struct MessagesOnHold { + /// The number of messages on hold. This is the sum of the length of `message_on_hold` + /// and the sum of the lengths of the values in `pending_ordered_messages`. + len: usize, + /// The size in bytes of the messages on hold, computed using [`Message::size`], computed + /// ontop of messages_on_hold, and also pending_ordered_messages. + size_in_bytes: usize, + /// Messages that ready to be processed by `pop`. + ready_messages: VecDeque, + /// Messages that are on hold for a given ordering key. + pending_ordered_messages: FxHashMap, VecDeque>, +} + +impl MessagesOnHold { + /// The number of messages on hold. + pub(crate) fn len(&self) -> usize { + self.len + } + + /// The size in bytes of the messages on hold, computed using [`Message::size`] + pub(crate) fn size_in_bytes(&self) -> usize { + self.size_in_bytes + } + + /// Returns how many messages on hold are ready to be handled. + pub(crate) fn num_ready_messages(&self) -> usize { + self.ready_messages.len() + } + + /// Pushes a message into the relevant messages on hold queue. + pub(crate) fn push(&mut self, message: Message) { + let queue_target = match &message.ordering_key { + // If the message has an ordering key, insert it into messages on hold if there is no queue for the ordering + // key and create a queue. Otherwise if the queue exists, insert the message into the respective queue. + Some(ordering_key) => { + match self.pending_ordered_messages.get_mut(ordering_key) { + Some(queue) => queue, + None => { + // We create a queue for this ording key, so that if we push another message + // with this given ordering key, it will be buffered within the queue, until + // `activate_ordering_key` is called. The presence of the queue means that we have + // a message this ordering key in the messages on hold queue. + self.pending_ordered_messages + .entry(ordering_key.clone()) + .or_default(); + + &mut self.ready_messages + } + } + } + None => &mut self.ready_messages, + }; + + self.len += 1; + self.size_in_bytes += message.size(); + queue_target.push_back(WrappedMessage(message)); + } + + /// Unwraps a message, decrementing the length and size of the MessagesOnHold struct. + fn unwrap_message(&mut self, message: WrappedMessage) -> Message { + let message = message.0; + + self.len -= 1; + self.size_in_bytes -= message.size(); + + message + } + + /// Gets a message from the on-hold queue. A message with an ordering + /// key wont be returned if there's another message with the same key in + /// flight. + #[inline(always)] + pub(crate) fn pop_ready(&mut self) -> Option { + self.ready_messages + .pop_front() + .map(|wrapped_message| self.unwrap_message(wrapped_message)) + } + + /// Internally moves an enqueued item from the front of given ordering key queue to the ready queue. + pub(crate) fn activate_ordering_key(&mut self, ordering_key: &str) { + if let Some(wrapped_message) = self.get_next_for_ordering_key(ordering_key) { + self.ready_messages.push_front(wrapped_message); + } + } + + /// Pops the next message for a given ordering key, removing the queue for a given ordering key, + /// if it is empty. + fn get_next_for_ordering_key(&mut self, ordering_key: &str) -> Option { + let queue_for_key = self.pending_ordered_messages.get_mut(ordering_key)?; + match queue_for_key.pop_front() { + // We have a message in the queue, return it and still preserve the queue. + Some(message) => Some(message), + // The queue is empty, we can clean it up now. + None => { + self.pending_ordered_messages.remove(ordering_key); + None + } + } + } + + /// Removes all messages from the messages on hold, + pub(crate) fn drain_and_collect_all_ack_ids(&mut self) -> Vec { + let mut ack_ids = Vec::with_capacity(self.len()); + + for message in self.ready_messages.drain(..) { + let (_, guard) = message.0.into_inner(); + ack_ids.push(guard.mark_acked_elsewhere()); + } + + for (_ordering_key, mut messages) in self.pending_ordered_messages.drain() { + for message in messages.drain(..) { + let (_, guard) = message.0.into_inner(); + ack_ids.push(guard.mark_acked_elsewhere()); + } + } + + // Reset these values, we should now be fully empty! + self.len = 0; + self.size_in_bytes = 0; + + debug_assert!(self.ready_messages.is_empty()); + debug_assert!(self.pending_ordered_messages.is_empty()); + + ack_ids + } +} diff --git a/osprey_coordinator/src/pub_sub_streaming_pull/metrics.rs b/osprey_coordinator/src/pub_sub_streaming_pull/metrics.rs new file mode 100644 index 0000000..f4178e6 --- /dev/null +++ b/osprey_coordinator/src/pub_sub_streaming_pull/metrics.rs @@ -0,0 +1,54 @@ +use crate::metrics::define_metrics; + +define_metrics!(StreamingPullManagerMetrics, [ + // How many messages the streaming pull subscriber has received. + messages_received => StaticCounter(), + // How many messages have been acked. + messages_acked => StaticCounter(), + // How many messages have been nacked. + messages_nacked => StaticCounter(), + // How many messages are currently buffered on hold. + messages_on_hold => StaticGauge(), + // How many bytes are on hold. + message_bytes_on_hold => StaticGauge(), + // How many messages are currently in-flight (aka processing) + messages_in_flight => StaticGauge(), + // How many messages (in terms of bytes), is currently in-flight (aka processing) + message_bytes_in_flight => StaticGauge(), + // How many messages were dispatched to the registered MessageHandler. + messages_dispatched_to_handler => StaticCounter(), + // Should be equivalent to the sum of messages on hold + messages in flight. + message_ack_queue_size => StaticGauge(), + // The current lease renewal duration that the streaming pull manager is using. + message_lease_renewal_duration_secs => StaticGauge(), + // How many message leases were renewed. + message_leases_renewed => StaticCounter(), + // How many background tasks the streaming pull manager is running (this is generally tasks spawned to handle + // background retries of requests to pub-sub). Should be 0 if everything is healthy. + background_tasks_running => StaticGauge(), + // How many streaming pull responses have been received from the server. + streaming_pull_response_received => StaticCounter(), + // Set to 1 when streaming pull has reached its flow control limits as negotiated with the server. + streaming_pull_flow_control_limited => StaticGauge(), + + // How long it takes once a message has been received for it to be acked. This counts + // time spent in the messages on hold either because there were too many messages + // in flight, or because ordering key activation. + ack_latency => StaticHistogram(), + // Same as ack_latency, but for nacks. + nack_latency => StaticHistogram(), + + // How many times the pub-sub streaming pull channel has abruptly ended. + pubsub_channel_disconnected => StaticCounter(), + // How many times the pub-sub streaming pull channel has encountered an error (while trying to connect). + pubsub_channel_connection_error => StaticCounter(), + + // Request latency of requests to the pub-sub API. + pubsub_grpc_request_latency => DynamicHistogram([request_type]), + // How many requests succeeded to the pub-sub API. + pubsub_grpc_request_success => DynamicCounter([request_type]), + // How many requests errored to the pub-sub api. + pubsub_grpc_request_error => DynamicCounter([request_type, code]), + // How many background requests errored so many times they were dropped. + pubsub_grpc_background_request_dropped => DynamicCounter([request_type]), +]); diff --git a/osprey_coordinator/src/pub_sub_streaming_pull/mod.rs b/osprey_coordinator/src/pub_sub_streaming_pull/mod.rs new file mode 100644 index 0000000..0f70d85 --- /dev/null +++ b/osprey_coordinator/src/pub_sub_streaming_pull/mod.rs @@ -0,0 +1,32 @@ +//! To get started, check out [`StreamingPullManager::new`]. +//! +//! # Example Usage +//! +//! For examples, see the `examples` directory in this crate. +#![warn(unused)] + +mod ack_id; +mod flow_control; +mod message; +mod message_ack_queue; +mod message_handler; +mod messages_in_flight; +mod messages_on_hold; +mod metrics; +mod streaming_pull_manager; +mod streaming_pull_manager_handle; + +pub(crate) use crate::pub_sub_streaming_pull::ack_id::AckId; +pub(crate) use crate::pub_sub_streaming_pull::message_ack_queue::{AckOrNack, MessageAckQueue}; +pub(crate) use crate::pub_sub_streaming_pull::messages_in_flight::MessagesInFlight; +pub(crate) use crate::pub_sub_streaming_pull::messages_on_hold::MessagesOnHold; +pub(crate) use crate::pub_sub_streaming_pull::metrics::StreamingPullManagerMetrics; +pub(crate) use crate::pub_sub_streaming_pull::streaming_pull_manager::StreamingPullManagerMessage; + +pub use crate::pub_sub_streaming_pull::flow_control::FlowControl; +pub use crate::pub_sub_streaming_pull::message::{DetachedMessage, Message}; +pub use crate::pub_sub_streaming_pull::message_handler::{ + MessageHandler, SpawnTaskPerMessageHandler, +}; +pub use crate::pub_sub_streaming_pull::streaming_pull_manager::StreamingPullManager; +pub use crate::pub_sub_streaming_pull::streaming_pull_manager_handle::StreamingPullManagerHandle; diff --git a/osprey_coordinator/src/pub_sub_streaming_pull/streaming_pull_manager.rs b/osprey_coordinator/src/pub_sub_streaming_pull/streaming_pull_manager.rs new file mode 100644 index 0000000..01d6ff2 --- /dev/null +++ b/osprey_coordinator/src/pub_sub_streaming_pull/streaming_pull_manager.rs @@ -0,0 +1,1048 @@ +use std::{ + future::{pending, Future}, + sync::Arc, + time::{Duration, Instant}, +}; + +use crate::backoff_utils::AsyncBackoff; +use crate::tokio_utils::AbortOnDrop; +use hdrhistogram::Histogram; +use tokio::{ + sync::{ + mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, + oneshot, + }, + task::JoinSet, + time::{interval, interval_at, MissedTickBehavior}, +}; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tonic::{ + codegen::{BoxFuture, InterceptedService}, + service::Interceptor, + Status, +}; + +use crate::gcloud::{ + google::pubsub::v1::{ + subscriber_client::SubscriberClient, AcknowledgeRequest, ModifyAckDeadlineRequest, + StreamingPullRequest, StreamingPullResponse, + }, + pubsub::PubSubSubscription, +}; +use crate::metrics::counters::StaticCounter; +use crate::metrics::emit_worker::SpawnEmitWorker; +use crate::metrics::gauges::StaticGauge; +use crate::metrics::histograms::StaticHistogram; +use crate::metrics::MetricsClientBuilder; +use crate::pub_sub_streaming_pull::{ + message::MessageAckGuard, AckId, AckOrNack, DetachedMessage, FlowControl, Message, + MessageAckQueue, MessageHandler, MessagesInFlight, MessagesOnHold, StreamingPullManagerHandle, + StreamingPullManagerMetrics, +}; + +/// The maximum number of ACK IDs to send in a single StreamingPullRequest. +const ACK_IDS_MAX_BATCH_SIZE: usize = 2500; + +/// How often we should update gauge based metrics. +const METRICS_REPORTING_INTERVAL: Duration = Duration::from_secs(1); + +/// How often we should attempt to flush leases. +const LEASE_RENEWAL_INTERVAL: Duration = Duration::from_secs(1); + +type TonicSubscriberClient = SubscriberClient>; + +impl FlowControl { + fn clamp_lease_duration(&self, lease_duration_secs: u32) -> u32 { + lease_duration_secs.clamp( + self.min_duration_per_lease_extension, + self.max_duration_per_lease_extension, + ) + } +} + +/// The [`StreamingPullManager`] is responsible for managing a connection to pub-sub, pulling messages over the +/// StreamingPull interface. The streaming pull manager runs a tokio task which maintains the connection to pub-sub +/// and handles re-connection, and lease/message management and buffering. +pub struct StreamingPullManager { + /// The recieve end of of [`StreamingPullManagerState::sender`], receives the messages and processes them. + receiver: UnboundedReceiver, + /// Internal state of the streaming pull manager. Refer to it's documentation. + state: StreamingPullManagerState, + /// The GRPC Streaming channel that pub-sub sends us messages on. + channel: StreamingPullChannel, + /// The [`MessageHandler`] that is invoked when messages are received and need to be processed. + message_handler: M, + /// Binds the lifetime of the metrics emitter to the lifetime of the streaming pull manager, which will cause the + /// metrics emitter to stop when the manager is stopped. + _metrics_emit_worker_abort_on_drop: AbortOnDrop, +} + +/// Holds the stateful bits of the streaming pull manager. +/// +/// This is a separate struct because the state is used to construct the channel, and thus we must have a state, +/// and channel before we can make the [`StreamingPullManager`]. See the `new` method's implementation. +struct StreamingPullManagerState { + /// The GRPC client that is used to communicate with pub-sub. + client: TonicSubscriberClient, + /// The flow control configuration. + flow_control: FlowControl, + /// How long the lease on messages should be (computed by clamping the min/max lease duration from flow control, + /// along with the p99 of message handling latency) + message_lease_renewal_duration_secs: u32, + /// The messages that are currently on hold for processing, either because too many messages are in-flight, or + /// we need to satisfy ordering_key constraints. + messages_on_hold: MessagesOnHold, + /// The client-id we are using to identify ourselves to pub-sub. This is a random Uuid that is generated at the + /// time the streaming pull manager is created. + client_id: uuid::Uuid, + /// The name of the pub-sub subscription we are trying to pull messages from, in the format: + /// `projects/{project}/subscriptions/{sub}`. + subscription: String, + /// The queue which holds all the ack-ids of messages received that are either on hold or processing, and batches + /// chunks of ack/nack requests to be flushed to pub-sub. + message_ack_queue: MessageAckQueue, + /// Mapping of AckId -> OrderingKey (for request activation), and to track which and how many requests are + /// currently in-flight. + messages_in_flight: MessagesInFlight, + /// When set to Some(...), the streaming pull manager has gracefully stopped. The join handle will resolve when + /// we have nacked all messages that were on hold with the pub-sub server. + graceful_stop_join_handles: Option, + /// A struct used to record metrics. + metrics: Arc, + /// Histogram for message latency, histogram is measured in milliseconds. + message_latency_histogram: Histogram, + /// A sender that is used to be [`Clone`]d into the [`MessageAckGuard`] part of a [`Message`], which is primarily + /// used to send an ack/nack message back to the manager, so it can send that back to pub-sub. Additionally, it is + /// used by the [`StreamingPullManagerHandle`] in order to stop the manager. + sender: UnboundedSender, + /// JoinSet of the background tasks that are currently running. + background_tasks: JoinSet<()>, + /// Async backoff to track retries for the pub-sub channel reconnection. + streaming_pull_channel_async_backoff: AsyncBackoff, +} + +struct GracefulStopJoinHandles { + message_handler: Option, +} + +impl StreamingPullManagerState +where + I: Interceptor + Clone + Send + Sync + 'static, +{ + /// Creates the initial streaming pull requests necessary to connect to the grpc channel! + /// + /// The initial streaming pull requests contain the flow control settings and grpc subscription to connect to. + fn initial_streaming_pull_request(&self) -> StreamingPullRequest { + StreamingPullRequest { + client_id: self.client_id.to_string(), + subscription: self.subscription.clone(), + max_outstanding_bytes: self.flow_control.max_bytes as _, + max_outstanding_messages: self.flow_control.max_messages as _, + stream_ack_deadline_seconds: self.message_lease_renewal_duration_secs as _, + ..Default::default() + } + } + + /// Performs the given request as a background task, retrying up to 10 times, with exponential backoff. + /// + /// Background task processing will block the streaming pull manager's graceful stop cycle. + fn perform_request_in_background_with_retries< + RequestExecutor, + ResponseResultOk, + RequestBody, + ResponseFuture, + >( + &mut self, + request_type: &'static str, + request_body: RequestBody, + request_executor: RequestExecutor, + ) where + ResponseFuture: Future> + Send, + ResponseResultOk: Send, + RequestBody: Clone + Send + Sync + tonic::IntoRequest + 'static, + RequestExecutor: Fn(TonicSubscriberClient, tonic::Request) -> ResponseFuture + + Send + + Sync + + 'static, + { + const MAX_BACKGROUND_REQUEST_ATTEMPTS: u8 = 10; + const BACKGROUND_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); + + // Process retries in the background, keeping track of in-flight background requests. + let subscription = self.subscription.clone(); + let client_id = self.client_id; + let client = self.client.clone(); + let metrics = self.metrics.clone(); + let mut async_backoff = async_backoff(); + let root_start_time = Instant::now(); + + self.background_tasks.spawn(async move { + for attempt in 1..=MAX_BACKGROUND_REQUEST_ATTEMPTS { + let start_time = Instant::now(); + let result = tokio::time::timeout( + BACKGROUND_REQUEST_TIMEOUT, + (request_executor)(client.clone(), request_body.clone().into_request()) + ) + .await + .map_err(|_| Status::deadline_exceeded("timeout exceeded")) + .and_then(|e| e); + + metrics.pubsub_grpc_request_latency.record_with_key( + request_type, + start_time.elapsed(), + ); + match result { + Ok(_) => { + metrics + .pubsub_grpc_request_success + .incr(request_type); + + tracing::debug!( + {%subscription, %client_id, %request_type}, + "background request {} has has succeeded on attempt {} ({:?} from when the request was intially attempted)", + request_type, attempt, root_start_time.elapsed(), + ); + return; + } + Err(error) => { + if attempt < MAX_BACKGROUND_REQUEST_ATTEMPTS { + metrics + .pubsub_grpc_request_error + .incr(request_type, error.code()); + + let retry_in = async_backoff.fail(); + tracing::error!( + {%subscription, %client_id, ?error, %request_type}, + "background request {} has failed on attempt {}, will retry in {:?}", + request_type, attempt, retry_in + ); + tokio::time::sleep(retry_in).await; + } else { + metrics.pubsub_grpc_background_request_dropped.incr(request_type); + + tracing::error!( + {%subscription, %client_id, ?error, %request_type}, + "background request {} has failed on attempt {}, giving up forever.", + request_type, attempt + ); + } + } + } + } + }); + } + + /// Tries to create the streaming pull channel. + /// + /// Does not retry! + fn create_streaming_pull_channel(&mut self, disconnected: bool) -> StreamingPullChannel { + let (sender, receiver) = unbounded_channel(); + self.recompute_message_lease_renewal_duration_secs(); + sender.send(self.initial_streaming_pull_request()).ok(); + + let mut client = self.client.clone(); + let subscription = self.subscription.clone(); + let client_id = self.client_id; + let connection_delay = if disconnected { + let retry_in = self.streaming_pull_channel_async_backoff.fail(); + tracing::info!( + {%subscription, %client_id}, + "will attempt to re-connect to the streaming pull channel in {:?}", + retry_in, + ); + retry_in + } else { + tracing::info!( + {%subscription, %client_id}, + "connecting to the streaming channel", + ); + Duration::ZERO + }; + + let connection_future = async move { + tokio::time::sleep(connection_delay).await; + + let start_time = Instant::now(); + let streaming = client + .streaming_pull(UnboundedReceiverStream::new(receiver)) + .await? + .into_inner(); + + tracing::info!( + {%subscription, %client_id}, + "connection to streaming pull channel received first message after {:?}", + start_time.elapsed() + ); + + Ok(streaming) + }; + + StreamingPullChannel::new(Box::pin(connection_future), sender) + } + + /// Returns the maximum amount of time we should wait before renewing the lease on a message. + fn get_buffered_lease_renewal_duration(&self) -> Duration { + // Allow for 20% buffer, or 5 seconds, which ever is greater. + let buffer = Duration::from_millis( + ((((self.message_lease_renewal_duration_secs * 1000) as f64) * 0.2) as u64).min(5_000), + ); + + Duration::from_secs(self.message_lease_renewal_duration_secs as _) - buffer + } + + /// Recomputes `message_lease_renewal_duration_secs` based upon the p99 of message processing latency. + fn recompute_message_lease_renewal_duration_secs(&mut self) { + let mut p99_message_latency_millis = + self.message_latency_histogram.value_at_percentile(99.0); + + // Round up to nearest second. + if p99_message_latency_millis % 1000 != 0 { + p99_message_latency_millis += 1000 - (p99_message_latency_millis % 1000); + } + + let next_lease_renewal_duration_secs = self + .flow_control + .clamp_lease_duration((p99_message_latency_millis / 1000) as _); + + if self.message_lease_renewal_duration_secs != next_lease_renewal_duration_secs { + tracing::info!( + {subscription = %self.subscription, client_id = %self.client_id}, + "updating message lease renewal duration secs from {} to {}", + self.message_lease_renewal_duration_secs, + next_lease_renewal_duration_secs + ); + self.message_lease_renewal_duration_secs = next_lease_renewal_duration_secs; + } + } +} + +/// Creates an [`AsyncBackoff`] for use where exponential backoff is needed with sensible defaults. +fn async_backoff() -> AsyncBackoff { + AsyncBackoff::new(crate::backoff_utils::Config { + min_delay: Duration::from_millis(100), + max_delay: Duration::from_secs(10), + ..Default::default() + }) +} + +impl StreamingPullManager +where + I: Interceptor + Clone + Send + Sync + 'static, + M: MessageHandler, +{ + /// Creates and starts a new [`StreamingPullManager`], returning a [`StreamingPullManagerHandle`] that can be used + /// to stop the manager. + /// + /// The function arguments are as follows: + /// - `client` the tonic grpc client that can make requests to the pub-sub GRPC server. + /// - `subscription` see [`PubSubSubscription`]. + /// - `flow_control` see [`FlowControl`]. + /// - `message_handler` see [`MessageHandler`], + /// - `metrics_client_builder` see [`MetricsClientBuilder`] + pub fn new( + client: TonicSubscriberClient, + subscription: PubSubSubscription, + flow_control: FlowControl, + message_handler: M, + metrics_client_builder: MetricsClientBuilder, + ) -> StreamingPullManagerHandle { + let client_id = uuid::Uuid::new_v4(); + + let metrics = StreamingPullManagerMetrics::new(); + let metrics_client = metrics_client_builder + .with_tag("subscription", subscription.to_string()) + .with_tag("client_id", client_id.to_string()) + .build() + .expect("invariant: failed to create metrics client"); + let _metrics_emit_worker_abort_on_drop = metrics.clone().spawn_emit_worker(metrics_client); + + let (sender, receiver) = unbounded_channel(); + let message_latency_histogram = Histogram::new_with_max( + (flow_control.max_duration_per_lease_extension * 1000) as _, + 3 + ).expect("invariant: histogram creation should never fail, as the bounds are checked in FlowControl"); + let message_lease_renewal_duration_secs = flow_control.min_duration_per_lease_extension; + let message_ack_queue = MessageAckQueue::new( + flow_control.max_messages, + std::cmp::min(ACK_IDS_MAX_BATCH_SIZE, flow_control.max_messages), + Duration::from_millis(100), + ); + + let mut state = StreamingPullManagerState { + client, + metrics, + client_id, + flow_control, + subscription: subscription.to_string(), + message_ack_queue, + message_latency_histogram, + message_lease_renewal_duration_secs, + sender: sender.clone(), + background_tasks: Default::default(), + messages_on_hold: Default::default(), + messages_in_flight: Default::default(), + graceful_stop_join_handles: Default::default(), + streaming_pull_channel_async_backoff: async_backoff(), + }; + + let channel = state.create_streaming_pull_channel(false); + let manager = Self { + state, + channel, + receiver, + message_handler, + _metrics_emit_worker_abort_on_drop, + }; + + StreamingPullManagerHandle::new( + subscription, + manager.state.client_id, + sender, + tokio::task::spawn(manager.main_loop()).into(), + ) + } + + /// Runs the main loop of the streaming pull manager. This runs inside a tokio task and is spawned by `new`. + async fn main_loop(mut self) -> Instant { + tracing::info!( + {subscription = %self.state.subscription, client_id = %self.state.client_id}, + "streaming pull manager starting." + ); + + enum Action { + /// Streaming pull caused an error, we'll need to re-connect to pub-sub. + StreamingPullError(Option), + /// Streaming pull returned a result that we need to process. + StreamingPullResult(StreamingPullResponse), + /// The mpsc to this task has received a message we need to process. + ManagerMessage(StreamingPullManagerMessage), + /// The message ack queue has asked us to flush a batch of ack_ids to pub_sub. + FlushAckOrNackChunk(Vec>), + /// We need to refresh our leases on messages that we currently are processing. + RenewLeases, + /// We need to report metrics. + ReportMetrics, + } + + let mut metrics_reporting_interval = interval(METRICS_REPORTING_INTERVAL); + metrics_reporting_interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + let mut lease_renewal_interval = interval(LEASE_RENEWAL_INTERVAL); + lease_renewal_interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + while !self.should_stop() { + // NOTE: We are keeping the bodies of the match arms in the select! block simple, because + // rust-analyzer autocomplete and syntax highlighting sucks inside of the tokio::select macro. + let action = tokio::select! { + // The ordering of each of the matches below is deliberate, and ranked in order of importance with + // the function that it performs and how urgent it is we must perform it when there is competing work. + // + // The rationale behind the ordering is explained below. + biased; + + // bias: metrics is a trivial task, and one that is important enough that in the event of resource + // starvation or contention, having metrics working will allow us to debug the system eaiser. + _ = metrics_reporting_interval.tick() => { + Action::ReportMetrics + } + // bias: Renewing leased messages is the most important to ensure we keep up on when we're under + // pressure. + _ = lease_renewal_interval.tick() => { + Action::RenewLeases + } + // bias: receiving messages from the mpsc channel is weighed over the message_ack_queue queue flushing, + // because it ensures that we fully empty the mpsc queue of all acks before we try to flush a chunk of + // acks to pub-sub. The send-end of the receiver is guaranteed to eventually stop producing messages, + // because we limit the number of messages we have in-flight at once. This means that we should never + // starve the tasks below. + Some(message) = self.receiver.recv() => { + Action::ManagerMessage(message) + } + // bias: it's important to handle acks/nacks before we read more messages from the pub-sub channel. + ack_batch = self.state.message_ack_queue.next_chunk() => { + Action::FlushAckOrNackChunk(ack_batch) + } + // bias: once all other tasks above are complete, we can receive messages from the pub-sub channel. + streaming_pull_result = self.channel.next_message() => { + match streaming_pull_result { + Ok(Some(response)) => Action::StreamingPullResult(response), + Ok(None) => Action::StreamingPullError(None), + Err(e) => Action::StreamingPullError(Some(e)) + } + }, + // bias: handle completion of background tasks, this is the lowest priority, and only used + // to report metrics, and determine when we should fully shutdown. + _ = self.state.background_tasks.join_next(), if !self.state.background_tasks.is_empty() => { + continue; + } + }; + + match action { + Action::ManagerMessage(StreamingPullManagerMessage::AckOrNack { + ack_or_nack, + latency, + }) => self.handle_ack_or_nack(ack_or_nack, latency), + Action::StreamingPullResult(streaming_pull_response) => { + self.handle_streaming_pull_response(streaming_pull_response) + } + Action::FlushAckOrNackChunk(chunk) => self.flush_ack_or_nack_chunk(chunk), + Action::RenewLeases => self.handle_renew_leases(), + Action::ReportMetrics => self.handle_report_metrics(), + Action::StreamingPullError(error) => self.handle_streaming_pull_error(error), + Action::ManagerMessage(StreamingPullManagerMessage::GracefulStop { sender }) => { + self.handle_graceful_stop(sender) + } + } + } + + tracing::info!( + {subscription = %self.state.subscription, client_id = %self.state.client_id}, + "all in-fight requests have been completed, stopping streaming pull manager" + ); + + // As part of graceful stop, we'll ensure that the graceful stop task has completed successfully, + // before exiting the manager loop. + if let Some(graceful_join_handle) = self.state.graceful_stop_join_handles.take() { + if let Some(message_handler) = graceful_join_handle.message_handler { + tracing::info!( + {subscription = %self.state.subscription, client_id = %self.state.client_id}, + "waiting for graceful stop tasks to complete" + ); + + message_handler + .await + .expect("message handler graceful stop task panicked"); + } + } + + tracing::info!( + {subscription = %self.state.subscription, client_id = %self.state.client_id}, + "streaming pull manager has stopped." + ); + + // Return the instant in which the streaming manager has stopped, which is used to fulfill the manager handle + // API (notably, tracking how long the graceful stop took.) + Instant::now() + } + + /// Determines whether or not we may be flow control limited from the server, by looking at the messages we + /// have on hold. + fn is_flow_control_limited(&self) -> bool { + self.state.messages_on_hold.len() > self.state.flow_control.max_messages + || self.state.messages_on_hold.size_in_bytes() > self.state.flow_control.max_bytes + } + + /// Handles a [`StreamingPullResponse`] that has been received from pub-sub. + /// + /// Immediate order of business is to mod-ack those messages to set their leases properly, and then will attempt + /// to schedule processing of the messages, respecting the `ordering_key` of the message, using [`MessagesOnHold`]. + fn handle_streaming_pull_response(&mut self, streaming_pull_response: StreamingPullResponse) { + self.state.streaming_pull_channel_async_backoff.succeed(); + self.state.metrics.streaming_pull_response_received.incr(); + self.state + .metrics + .messages_received + .incr_by(streaming_pull_response.received_messages.len() as _); + + self.transform_and_insert_messages_into_messages_on_hold(streaming_pull_response); + self.try_flush_messages_on_hold_batch(); + } + + /// Handles a message being acked or nacked, enqueueing the ack/nack within the [`MessageAckQueue`] and + /// scheduling any messages on hold that would have been activated by the completion of the message (if it has + /// an `ordering_key`). + fn handle_ack_or_nack(&mut self, ack_or_nack: AckOrNack, latency: Duration) { + match &ack_or_nack { + AckOrNack::Ack(_) => { + self.state.metrics.messages_acked.incr(); + self.state.metrics.ack_latency.record(latency) + } + AckOrNack::Nack(_) => { + self.state.metrics.messages_nacked.incr(); + self.state.metrics.nack_latency.record(latency) + } + } + + self.state + .message_latency_histogram + .saturating_record(latency.as_millis() as _); + + let ack_id = *ack_or_nack.ack_id(); + self.state + .message_ack_queue + .enqueue_ack_or_nack(ack_or_nack); + + if let Some(Some(ordering_key)) = self.state.messages_in_flight.remove(&ack_id) { + self.state + .messages_on_hold + .activate_ordering_key(&ordering_key); + } + + // When a message is acked, or nacked, the message is removed from `messages_in_flight`, + // this means that we may be able to process more messages, so we should try to flush a + // batch. + self.try_flush_messages_on_hold_batch(); + } + + /// Transforms the pub-sub messages into [`Message`]s, and inserts them into the [`MessagesOnHold`]. + fn transform_and_insert_messages_into_messages_on_hold( + &mut self, + streaming_pull_response: StreamingPullResponse, + ) { + tracing::debug!( + {subscription = %self.state.subscription, client_id = %self.state.client_id}, + "received {} messages from pub-sub", + streaming_pull_response.received_messages.len() + ); + + let received_at = Instant::now(); + // The rationale of this being 1 second from now is that we'll delay these by 1 second so that we can build + // up a batch of lease renewals within the message ack queue's lease delay queue. + let lease_renew_at = tokio::time::Instant::from(received_at + Duration::from_secs(1)); + + streaming_pull_response + .received_messages + .into_iter() + .filter_map(|received_message| { + let proto_message = received_message.message?; + Some(Message::new( + DetachedMessage::new(proto_message, received_message.delivery_attempt as _), + MessageAckGuard::new( + self.state.sender.clone(), + received_at, + self.state.message_ack_queue.transform_and_store_ack_id( + received_message.ack_id, + received_at, + lease_renew_at, + ), + ), + )) + }) + .for_each(|message| { + self.state.messages_on_hold.push(message); + }); + } + + /// Attempts to flush messages from [`MessagesOnHold`] to the [`MessageHandler`]. + fn try_flush_messages_on_hold_batch(&mut self) { + // How many more messages we can process until we hit the max_processing_messages limit. + let num_additional_messages_that_can_be_processed = self + .state + .flow_control + .max_processing_messages + .saturating_sub(self.state.messages_in_flight.len()); + + // This has the fun property of calling Vec::with_capacity(0), which will not allocate on the heap + // if there are no additional messages to process, and precisely allocating a vec sized to the number + // of messages we'll be handling in this batch. + let mut messages = Vec::with_capacity(std::cmp::min( + self.state.messages_on_hold.num_ready_messages(), + num_additional_messages_that_can_be_processed, + )); + + while self.state.messages_in_flight.len() < self.state.flow_control.max_processing_messages + { + let message = match self.state.messages_on_hold.pop_ready() { + Some(message) => message, + None => break, + }; + + self.state.messages_in_flight.insert(&message); + + messages.push(message); + } + + if messages.is_empty() { + return; + } + + tracing::debug!( + {subscription = %self.state.subscription, client_id = %self.state.client_id}, + "submitting {} messages to message handler", + messages.len() + ); + + self.state + .metrics + .messages_dispatched_to_handler + .incr_by(messages.len() as _); + + self.message_handler.handle_messages(messages); + } + + /// Reports metric gauges. + fn handle_report_metrics(&self) { + self.state + .metrics + .messages_on_hold + .set(self.state.messages_on_hold.len() as _); + + self.state + .metrics + .message_bytes_on_hold + .set(self.state.messages_on_hold.size_in_bytes() as _); + + self.state + .metrics + .message_ack_queue_size + .set(self.state.message_ack_queue.len() as _); + + self.state + .metrics + .messages_in_flight + .set(self.state.messages_in_flight.len() as _); + + self.state + .metrics + .message_bytes_in_flight + .set(self.state.messages_in_flight.bytes_in_flight() as _); + + self.state + .metrics + .background_tasks_running + .set(self.state.background_tasks.len() as _); + + self.state + .metrics + .message_lease_renewal_duration_secs + .set(self.state.message_lease_renewal_duration_secs as _); + + self.state + .metrics + .streaming_pull_flow_control_limited + .set(if self.is_flow_control_limited() { 1 } else { 0 }); + } + + /// Begins graceful stop process by closing out the channel, and nacking all un-processed messages. + fn handle_graceful_stop( + &mut self, + sender: oneshot::Sender>, + ) { + if self.state.graceful_stop_join_handles.is_some() { + sender.send(Err(GracefulStopAlreadyRequested)).ok(); + return; + } + + tracing::info!( + {subscription = %self.state.subscription, client_id = %self.state.client_id}, + "graceful stop requested" + ); + // Graceful stop first closes the channel to pub-sub, which should prevent google from sending us any more + // messages. + self.channel.close(); + + // Now, we need to drain all the messages we have yet to submit to the message handler. + let ack_or_nacks = self.state.messages_on_hold.drain_and_collect_all_ack_ids(); + + // Create the mod-ack requests we need in order to release our lease on the messages we had on hold. + for chunk in ack_or_nacks.chunks(ACK_IDS_MAX_BATCH_SIZE) { + let chunk = chunk + .into_iter() + .filter_map(|ack_id| self.state.message_ack_queue.remove_ack_id(ack_id)) + .collect(); + + let modack_request = make_modack_request(&self.state.subscription, chunk, 0); + if modack_request.ack_ids.is_empty() { + continue; + } + + self.state.perform_request_in_background_with_retries( + "nack_messages_on_hold", + modack_request, + |mut c, r| async move { c.modify_ack_deadline(r).await }, + ) + } + + let message_handler_graceful_stop = self.message_handler.handle_graceful_stop(); + self.state.graceful_stop_join_handles = Some(GracefulStopJoinHandles { + message_handler: message_handler_graceful_stop, + }); + sender.send(Ok(())).ok(); + } + + /// Returns `true` if the manager's main loop should exit. + /// + /// Checks to make sure all in-flight messages and acks have been flushed after a graceful stop was requested. + pub(crate) fn should_stop(&self) -> bool { + if self.state.graceful_stop_join_handles.is_none() { + return false; + } + + self.state.messages_in_flight.is_empty() + && self.state.message_ack_queue.is_empty() + && self.state.background_tasks.is_empty() + } + + /// Issues modack requests to pub-sub in order to renew leases for messages that are either in-flight or on hold. + /// + /// Returns true if a full batch was flushed, and the next lease renewal flush should perhaps be expedited. + fn handle_renew_leases(&mut self) { + loop { + self.state.recompute_message_lease_renewal_duration_secs(); + let lease_renewal_duration = + Duration::from_secs(self.state.message_lease_renewal_duration_secs as _); + + let chunk = self + .state + .message_ack_queue + .collect_ack_ids_that_need_to_be_renewed( + ACK_IDS_MAX_BATCH_SIZE, + self.state.get_buffered_lease_renewal_duration(), + ); + + if chunk.is_empty() { + break; + } + + let chunk_is_full = chunk.len() == ACK_IDS_MAX_BATCH_SIZE; + + tracing::debug!( + {subscription = %self.state.subscription, client_id = %self.state.client_id}, + "renewing leases for {} messages with a lease duration of {} seconds", + chunk.len(), + self.state.message_lease_renewal_duration_secs, + ); + + self.state + .metrics + .message_leases_renewed + .incr_by(chunk.len() as _); + + let mut ack_ids = Vec::with_capacity(chunk.len()); + + for (elapsed, ack_id) in chunk { + if elapsed > lease_renewal_duration { + self.state + .message_latency_histogram + .saturating_record(elapsed.as_millis() as _); + } + ack_ids.push(ack_id); + } + + self.state.perform_request_in_background_with_retries( + "renew_leases", + make_modack_request( + &self.state.subscription, + ack_ids, + self.state.message_lease_renewal_duration_secs, + ), + |mut c, r| async move { c.modify_ack_deadline(r).await }, + ); + + if !chunk_is_full { + break; + } + } + } + + /// Flushes a chunk of acks or nacks to pub-sub server. + fn flush_ack_or_nack_chunk(&mut self, ack_or_nack_chunk: Vec>) { + let (ack_request, nack_request) = + make_ack_messages(&self.state.subscription, ack_or_nack_chunk); + + if let Some(ack_request) = ack_request { + tracing::debug!( + {subscription = %self.state.subscription, client_id = %self.state.client_id}, + "acking {} messages.", + ack_request.ack_ids.len() + ); + + self.state.perform_request_in_background_with_retries( + "ack", + ack_request, + |mut c, r| async move { c.acknowledge(r).await }, + ); + } + + if let Some(nack_request) = nack_request { + tracing::debug!( + {subscription = %self.state.subscription, client_id = %self.state.client_id}, + "nacking {} messages.", + nack_request.ack_ids.len() + ); + self.state.perform_request_in_background_with_retries( + "nack", + nack_request, + |mut c, r| async move { c.modify_ack_deadline(r).await }, + ); + } + } + + fn handle_streaming_pull_error(&mut self, error: Option) { + match error { + Some(error) => { + tracing::error!( + {subscription = %self.state.subscription, client_id = %self.state.client_id, ?error}, + "streaming pull channel encountered an error, will reconnect." + ) + } + None => { + tracing::error!( + {subscription = %self.state.subscription, client_id = %self.state.client_id}, + "streaming pull channel abruptly ended, will reconnect." + ) + } + } + self.state.metrics.pubsub_channel_disconnected.incr(); + self.channel = self.state.create_streaming_pull_channel(true); + } +} + +fn make_modack_request( + subscription: &str, + ack_ids: Vec, + deadline_secs: u32, +) -> ModifyAckDeadlineRequest { + ModifyAckDeadlineRequest { + subscription: subscription.to_string(), + ack_deadline_seconds: deadline_secs as _, + ack_ids, + } +} + +/// Makes messages that need to be sent to flush the pending ack/nacks. +fn make_ack_messages( + subscription: &str, + ack_chunks: Vec>, +) -> (Option, Option) { + let mut num_acks = 0; + let mut num_nacks = 0; + for ack_or_nack in &ack_chunks { + match ack_or_nack { + AckOrNack::Ack(_) => num_acks += 1, + AckOrNack::Nack(_) => num_nacks += 1, + } + } + + let mut ack_ids = Vec::with_capacity(num_acks); + let mut nack_ids = Vec::with_capacity(num_nacks); + for ack_or_nack in ack_chunks { + match ack_or_nack { + AckOrNack::Ack(ack_id) => ack_ids.push(ack_id), + AckOrNack::Nack(ack_id) => nack_ids.push(ack_id), + } + } + + let ack_request = (!ack_ids.is_empty()).then(|| AcknowledgeRequest { + subscription: subscription.to_string(), + ack_ids, + }); + + let nack_request = (!nack_ids.is_empty()).then(|| ModifyAckDeadlineRequest { + subscription: subscription.to_string(), + ack_ids: nack_ids, + ack_deadline_seconds: 0, + }); + + (ack_request, nack_request) +} + +pub(crate) enum StreamingPullManagerMessage { + AckOrNack { + ack_or_nack: AckOrNack, + latency: Duration, + }, + GracefulStop { + sender: oneshot::Sender>, + }, +} + +/// Graceful stop has already been requested. +#[derive(Debug)] +pub(crate) struct GracefulStopAlreadyRequested; + +/// Holds the tonic streaming channel, and the sender to push messages into that bi-directional channel. +pub enum StreamingPullChannel { + /// Hackfix for: https://github.com/hyperium/tonic/issues/515 + Connecting { + future: BoxFuture, tonic::Status>, + heartbeater: Option, + }, + /// The channel is connected. + Connected { + stream: tonic::Streaming, + #[allow(unused)] + heartbeater: AbortOnDrop, + }, + /// The channel has been closed by [`StreamingPullChannel::close`]. + /// + /// This should only occur during shutdown. + Closed, +} + +impl StreamingPullChannel { + fn new( + future: BoxFuture, tonic::Status>, + sender: UnboundedSender, + ) -> Self { + /// How often to send heartbeats in seconds. Determined as half the period of + /// time where the Pub/Sub server will close the stream as inactive, which is + /// 60 seconds. + const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30); + + // The heartbeater is deliberately *not* run in the streaming pull manager, incase it becomes backlogged + // on work, we still want heartbeats to proceed accordingly. + let heartbeater = tokio::task::spawn(async move { + let mut interval = interval_at( + tokio::time::Instant::now() + HEARTBEAT_INTERVAL, + HEARTBEAT_INTERVAL, + ); + interval.set_missed_tick_behavior(MissedTickBehavior::Delay); + + loop { + interval.tick().await; + + // We simply send an empty StreamingPullRequest when the heartbeat needs to be renewed. + if sender.send(StreamingPullRequest::default()).is_err() { + break; + } + } + }) + .into(); + + Self::Connecting { + future, + heartbeater: Some(heartbeater), + } + } + + /// Closes the channel, and makes it so that `next_message` will never resolve. + fn close(&mut self) { + // Close the stream by simply dropping it. There doesn't appear to be any more method within tonic + // that will explicitly "close" the stream. We simply rely on the drop handler to clean things up. + *self = Self::Closed; + } + + /// Polls for the next message. If the underlying stream is closed, then this method will simply + /// never resolve and be pending perpetually. + /// + /// NOTE: This method is cancel-safe. Cancelling this future will not cause messages to be lost. + async fn next_message(&mut self) -> Result, tonic::Status> { + loop { + match self { + StreamingPullChannel::Connected { stream, .. } => return stream.message().await, + StreamingPullChannel::Connecting { + future, + heartbeater, + } => { + match future.await { + Ok(stream) => { + let heartbeater = heartbeater + .take() + .expect("invariant: heartbeater should exist"); + *self = StreamingPullChannel::Connected { + stream, + heartbeater, + }; + } + Err(status) => { + *self = StreamingPullChannel::Closed; + return Err(status); + } + }; + } + StreamingPullChannel::Closed => return pending().await, + } + } + } +} diff --git a/osprey_coordinator/src/pub_sub_streaming_pull/streaming_pull_manager_handle.rs b/osprey_coordinator/src/pub_sub_streaming_pull/streaming_pull_manager_handle.rs new file mode 100644 index 0000000..d003a06 --- /dev/null +++ b/osprey_coordinator/src/pub_sub_streaming_pull/streaming_pull_manager_handle.rs @@ -0,0 +1,512 @@ +use std::{ + any::Any, + future::Future, + time::{Duration, Instant}, +}; + +use crate::gcloud::pubsub::PubSubSubscription; +use crate::tokio_utils::AbortOnDrop; +use tokio::sync::{mpsc::UnboundedSender, oneshot}; + +use crate::pub_sub_streaming_pull::StreamingPullManagerMessage; + +/// Returned by [`crate::StreamingPullManager::new`]. Can be used to stop the manager. +/// +/// If this manager handle is dropped, the streaming pull manager is immediately hard stopped. +#[must_use] +pub struct StreamingPullManagerHandle { + sender: UnboundedSender, + client_id: uuid::Uuid, + subscription: PubSubSubscription, + state: HandleState, +} + +#[derive(Debug)] +enum HandleState { + Running(AbortOnDrop), + GracefullyStopped(Instant), + PreviouslyPanicked, +} + +impl HandleState { + fn take_abort_on_drop(&mut self) -> Option> { + let taken = std::mem::replace(self, Self::PreviouslyPanicked); + match taken { + Self::Running(abort_on_drop) => Some(abort_on_drop), + _ => None, + } + } + + async fn join(&mut self) -> Result { + match self { + Self::Running(abort_on_drop) => { + let result = abort_on_drop.await; + match result { + Ok(instant) => { + *self = Self::GracefullyStopped(instant); + Ok(instant) + } + Err(join_error) => { + *self = Self::PreviouslyPanicked; + Err(StreamingPullManagerPanicError::Panicked { + panic: join_error.into_panic(), + }) + } + } + } + Self::GracefullyStopped(instant) => Ok(*instant), + Self::PreviouslyPanicked => Err(StreamingPullManagerPanicError::PreviouslyPanicked), + } + } +} + +impl StreamingPullManagerHandle { + pub(crate) fn new( + subscription: PubSubSubscription, + client_id: uuid::Uuid, + sender: UnboundedSender, + manager_abort_on_drop: AbortOnDrop, + ) -> Self { + Self { + sender, + client_id, + subscription, + state: HandleState::Running(manager_abort_on_drop), + } + } + + pub fn client_id(&self) -> uuid::Uuid { + self.client_id + } + + pub fn subscription(&self) -> &PubSubSubscription { + &self.subscription + } + + /// Attempts to gracefully stop the manager. The graceful stop might fail because the manager had panicked already. + /// + /// Returns a [`StreamingPullManagerStoppingHandle`] which can be used to hard stop the manager after a specific + /// amount of time has passed. + pub async fn graceful_stop( + mut self, + ) -> Result { + let abort_on_drop = match self.state.take_abort_on_drop() { + Some(abort_on_drop) => abort_on_drop, + None => return Err(StreamingPullManagerPanicError::PreviouslyPanicked), + }; + + let (sender, receiver) = oneshot::channel(); + let graceful_stop_initiated_at = Instant::now(); + + tracing::info!( + {subscription = %self.subscription, client_id = %self.client_id}, + "streaming pull manager handler requested graceful stop" + ); + + self.sender + .send(StreamingPullManagerMessage::GracefulStop { sender }) + .ok(); + + match receiver.await { + // The streaming pull manager has acknowledged our request to gracefully stop. + Ok(Ok(())) => Ok(StreamingPullManagerStoppingHandle { + graceful_stop_initiated_at, + state: HandleState::Running(abort_on_drop), + client_id: self.client_id, + subscription: self.subscription + }), + // This should never happen, but we can't prove this to rust. + Ok(Err(_)) => unreachable!( + "invariant: there can not be another graceful stop requested, + because graceful_stop can only be called once (because the graceful stop method is consuming)" + ), + // The streaming pull manager had dropped our oneshot, that means that it must have stopped, figure out why: + Err(_) => match abort_on_drop.await { + // The streaming pull manager had already cleanly stopped, but at who's behest? + Ok(_) => unreachable!( + "invariant: the streaming pull manager had cleanly stopped for an unknown reason?" + ), + // The streaming pull manager must have panicked :(. + Err(join_error) => Err(StreamingPullManagerPanicError::Panicked { + panic: join_error.into_panic(), + }), + }, + } + } + + /// Monitors the manager, resolving if the manager unexpectedly stops. + /// + /// This is useful to detect if the manager has panicked, and to cascade the failure to the rest of the system. + /// + /// # Cancel Safety: + /// + /// This function is cancel safe, and will not lose any error states if the future is cancelled prior to completion. + pub async fn monitor(&mut self) -> StreamingPullManagerPanicError { + match self.state.join().await { + // The streaming pull manager stopped, for unknown reason. This is a bug. + Ok(_) => unreachable!( + "invariant: the streaming pull manager had cleanly stopped, without us calling `graceful_stop`", + ), + // The streaming pull manager must have panicked. + Err(error) => error, + } + } + + /// This is a convenience method which will provide a ctrl-c handler to stop a streaming pull manager. + /// + /// This method writes output to stderr instead of using the tracing logging facilities, as this ctrl-c handling is + /// generally done when developing a service. This method should not be used in production, as it does not handle + /// SIGTERM, which is usually what kubernetes and other job managers will use to signal the job to stop. + /// + /// The stop procedure is as follows: + /// 1. Wait for ctrl-c, or the manager to unexpectedly panic. If the manager had panicked, propagate that panic. + /// 2. Initiate a graceful stop of the manager. + /// 3. Wait till either `graceful_stop_timeout` has elapsed, or ctrl-c is hit again. + /// 4. Hard stop the manager. + pub async fn gracefully_stop_on_ctrl_c( + mut self, + graceful_stop_timeout: Duration, + ) -> StreamingPullManagerStopped { + tokio::select! { + _ = tokio::signal::ctrl_c() => {}, + error = self.monitor() => error.continue_panicking() + } + + eprintln!( + "[streaming pull manager] ctrl-c received, gracefully stopping streaming pull manager (subscription={}, client_id={})", + self.subscription, self.client_id + ); + + let mut stopping = match self.graceful_stop().await { + Ok(stopping) => stopping, + Err(error) => error.continue_panicking(), + }; + + eprintln!( + "[streaming pull manager] graceful stop has begun. will wait {:?} until hard stopping. press ctrl-c again to hard stop immediately.", + graceful_stop_timeout, + ); + + enum SelectResult { + StopResult( + Result, + ), + StopNow { + ctrl_c: bool, + }, + } + + let select_result = tokio::select! { + stop_result = stopping.join() => SelectResult::StopResult(stop_result), + _ = tokio::time::sleep(graceful_stop_timeout) => SelectResult::StopNow { ctrl_c: false }, + _ = tokio::signal::ctrl_c() => SelectResult::StopNow { ctrl_c: true }, + }; + + match select_result { + SelectResult::StopResult(Ok(gracefully)) => { + eprintln!( + "[streaming pull manager] graceful stop has succeeded, took {:?}", + gracefully.duration + ); + gracefully.into() + } + SelectResult::StopResult(Err(panicked)) => panicked.continue_panicking(), + SelectResult::StopNow { ctrl_c } => { + if ctrl_c { + eprintln!("[streaming pull manager] ctrl-c hit again. hard stopping now."); + } else { + eprintln!( + "[streaming pull manager] {:?} has elapsed, hard stopping now.", + graceful_stop_timeout + ); + } + + match stopping.hard_stop_now().await { + Ok(StreamingPullManagerStopped::Gracefully(gracefully)) => { + eprintln!( + "[streaming pull manager] even though a hard stop was requested, the streaming pull manager beat us to the punch and gracefully stopped, took: {:?}", + gracefully.duration + ); + gracefully.into() + } + Ok(StreamingPullManagerStopped::Hard(hard)) => { + eprintln!( + "[streaming pull manager] hard stop completed, took {:?}", + hard.duration + ); + hard.into() + } + Err(panicked) => panicked.continue_panicking(), + } + } + } + } + + /// This is a convenience method which will gracefully stop the streaming pull manager when the provided future + /// resolves, waiting up to `graceful_stop_timeout` before hard stopping the manager. + /// + /// This method will panic if the streaming pull manager panics. + pub async fn gracefully_stop_on_signal( + mut self, + signal: impl Future, + graceful_stop_timeout: Duration, + ) -> StreamingPullManagerStopped { + tokio::select! { + _ = signal => {}, + error = self.monitor() => error.continue_panicking() + } + + let subscription = self.subscription.clone(); + let client_id = self.client_id; + + tracing::warn!( + {%subscription, %client_id}, + "signal received, gracefully stopping streaming pull manager", + ); + + let stopping = match self.graceful_stop().await { + Ok(stopping) => stopping, + Err(error) => error.continue_panicking(), + }; + + tracing::warn!( + {%subscription, %client_id}, + "graceful stop has begun. will wait {:?} until hard stopping.", + graceful_stop_timeout, + ); + + match stopping.hard_stop_after(graceful_stop_timeout).await { + Err(error) => error.continue_panicking(), + Ok(StreamingPullManagerStopped::Gracefully(gracefully)) => { + tracing::warn!( + {%subscription, %client_id}, + "gracefullly stopped in {:?}", + gracefully.duration, + ); + gracefully.into() + } + Ok(StreamingPullManagerStopped::Hard(hard)) => { + tracing::warn!( + {%subscription, %client_id}, + "hard stopped in {:?} (hard stop took {:?} to process)", + hard.duration, hard.abort_duration + ); + hard.into() + } + } + } +} + +#[derive(Debug)] +pub enum StreamingPullManagerPanicError { + /// Streaming pull manager has panicked. + Panicked { + /// The panic that was thrown from the streaming pull manager. + panic: Box, + }, + /// The streaming pull manager had panicked in the past, and [`StreamingPullManagerPanicError::Panicked`] was + /// returned. + PreviouslyPanicked, +} + +impl StreamingPullManagerPanicError { + /// Continues the panic that started in the manager. + pub fn continue_panicking(self) -> ! { + match self { + StreamingPullManagerPanicError::Panicked { panic } => std::panic::resume_unwind(panic), + StreamingPullManagerPanicError::PreviouslyPanicked => { + panic!("manager had previously panicked") + } + } + } +} + +impl std::fmt::Display for StreamingPullManagerPanicError { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + StreamingPullManagerPanicError::PreviouslyPanicked => { + write!(fmt, "previously panicked") + } + StreamingPullManagerPanicError::Panicked { .. } => { + write!(fmt, "panicked") + } + } + } +} + +impl std::error::Error for StreamingPullManagerPanicError {} + +#[derive(Debug)] +#[must_use] +pub struct StreamingPullManagerStoppingHandle { + client_id: uuid::Uuid, + subscription: PubSubSubscription, + graceful_stop_initiated_at: Instant, + state: HandleState, +} + +#[derive(Debug)] +pub struct StreamingPullManagerGracefullyStopped { + /// How long it took for the graceful stop to complete, measured as the time spent since + /// [`StreamingPullManagerHandle::graceful_stop`] was called. + pub duration: Duration, + /// The instant in which the graceful stop had completed. + pub instant: Instant, +} + +impl StreamingPullManagerGracefullyStopped { + fn new(graceful_stop_started_at: Instant, graceful_stop_completed_at: Instant) -> Self { + Self { + duration: graceful_stop_completed_at + .saturating_duration_since(graceful_stop_started_at), + instant: graceful_stop_completed_at, + } + } +} + +#[derive(Debug)] +pub struct StreamingPullManagerHardStopped { + /// How long it took for the stop to complete, measured as the time spent since + /// [`StreamingPullManagerHandle::graceful_stop`] was called. + pub duration: Duration, + /// How long between when we tried to `abort` the StreamingPullManager, and when it fully stopped. + pub abort_duration: Duration, + /// The instant in which the hard stop had completed. + pub instant: Instant, +} + +#[derive(Debug)] +pub enum StreamingPullManagerStopped { + /// The streaming pull manager has successfully gracefully stopped. + Gracefully(StreamingPullManagerGracefullyStopped), + /// The streaming pull manager has successfully hard stopped. + Hard(StreamingPullManagerHardStopped), +} + +impl From for StreamingPullManagerStopped { + fn from(stopped: StreamingPullManagerGracefullyStopped) -> Self { + Self::Gracefully(stopped) + } +} + +impl From for StreamingPullManagerStopped { + fn from(stopped: StreamingPullManagerHardStopped) -> Self { + Self::Hard(stopped) + } +} + +impl StreamingPullManagerStoppingHandle { + pub fn client_id(&self) -> uuid::Uuid { + self.client_id + } + + pub fn subscription(&self) -> &PubSubSubscription { + &self.subscription + } + + /// Attempts to hard-stop the manager if the graceful stop does not complete after `graceful_stop_timeout` + /// has elapsed. + /// + /// See [`StreamingPullManagerStoppingHandle::hard_stop_now`] for more details. + pub async fn hard_stop_after( + mut self, + graceful_stop_timeout: Duration, + ) -> Result { + tracing::warn!( + {subscription = %self.subscription, client_id = %self.client_id}, + "streaming pull manager hard stop requested with a graceful stop timeout of {:?}", + graceful_stop_timeout + ); + + let stop_status = match tokio::time::timeout(graceful_stop_timeout, self.join()).await { + Ok(join_result) => StreamingPullManagerStopped::Gracefully(join_result?), + Err(_) => self.hard_stop_now().await?, + }; + + Ok(stop_status) + } + + /// Attempts to hard stop the streaming pull manager immediately. + /// + /// A hard stop will abort the task that is running the manager, causing it to immediately stop. + pub async fn hard_stop_now( + self, + ) -> Result { + let join_handle = match self.state { + HandleState::Running(abort_on_drop) => abort_on_drop.into_inner(), + HandleState::GracefullyStopped(instant) => { + return Ok(StreamingPullManagerGracefullyStopped::new( + self.graceful_stop_initiated_at, + instant, + ) + .into()); + } + HandleState::PreviouslyPanicked => { + return Err(StreamingPullManagerPanicError::PreviouslyPanicked) + } + }; + + tracing::warn!( + {subscription = %self.subscription, client_id = %self.client_id}, + "streaming pull hard stopping now, aborting manager task", + ); + + let abort_started_at = Instant::now(); + join_handle.abort(); + + let join_error = match join_handle.await { + // The task had already completed successfully. + Ok(instant) => { + tracing::warn!( + {subscription = %self.subscription, client_id = %self.client_id}, + "streaming pull manager task was aborted, however, it had completed prior to the abort request being processed", + ); + + return Ok(StreamingPullManagerGracefullyStopped::new( + self.graceful_stop_initiated_at, + instant, + ) + .into()); + } + // The task may have panicked, or successfully aborted itself. + Err(join_error) => join_error, + }; + + match join_error.try_into_panic() { + Ok(panic) => Err(StreamingPullManagerPanicError::Panicked { panic }), + // The join error is due to us calling `join_handle.abort()`. + Err(_) => { + let instant = Instant::now(); + let abort_duration = instant - abort_started_at; + let hard = StreamingPullManagerHardStopped { + duration: self.graceful_stop_initiated_at.elapsed(), + abort_duration, + instant, + }; + + tracing::warn!( + {subscription = %self.subscription, client_id = %self.client_id}, + "streaming pull manager task was successfully hard stopped, took {:?} since graceful start, and {:?} to process the hard stop", + hard.duration, hard.abort_duration + ); + + Ok(hard.into()) + } + } + } + + /// Waits for the manager to gracefully stop. + /// + /// The manager may panic before then however, in which case an error is returned. + /// + /// # Cancel Safety: + /// + /// This function is cancel safe, and will not lose any error states if the future is cancelled prior to completion. + pub async fn join( + &mut self, + ) -> Result { + self.state.join().await.map(|instant| { + StreamingPullManagerGracefullyStopped::new(self.graceful_stop_initiated_at, instant) + }) + } +} diff --git a/osprey_coordinator/src/pubsub.rs b/osprey_coordinator/src/pubsub.rs new file mode 100644 index 0000000..1a41c23 --- /dev/null +++ b/osprey_coordinator/src/pubsub.rs @@ -0,0 +1,496 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use crate::gcloud::grpc::connection::Connection; +use crate::gcloud::{ + auth::AuthorizationHeaderInterceptor, + gcp_metadata::GCPMetadataClient, + google::pubsub::v1::subscriber_client::SubscriberClient, + kms::{AesGcmEnvelope, GOOGLE_KMS_DOMAIN}, + pubsub::{PubSubSubscription, GOOGLE_PUBSUB_DOMAIN}, +}; +use crate::metrics::counters::StaticCounter; +use crate::metrics::histograms::StaticHistogram; +use crate::metrics::MetricsClientBuilder; +use crate::{ + coordinator_metrics::SmiteCoordinatorMetrics, + priority_queue::{AckOrNack, AckableAction, PriorityQueueSender}, + proto::{self, smite_coordinator_action::SecretData}, + pub_sub_streaming_pull::DetachedMessage, + pub_sub_streaming_pull::{FlowControl, SpawnTaskPerMessageHandler, StreamingPullManager}, +}; +use anyhow::{anyhow, Result}; +use msgpack_simple::MsgPack; +use prost::Message; +use prost_types::Timestamp; +use rand::Rng; +use serde::Deserialize; +use serde_json::Value; +use tokio::time::{timeout, Duration as TokioDuration, Instant}; +use tonic::{codegen::InterceptedService, transport::Channel}; + +use crate::proto::Action as SmiteProtoAction; +use crate::signals::exit_signal; +use crate::snowflake_client::SnowflakeClient; +use convert_case::{Case, Casing}; +use proto::smite_coordinator_action::ActionData; + +async fn decode_proto_message( + message_data: &[u8], + ack_id: u64, + message_timestamp: Timestamp, + snowflake_client: &SnowflakeClient, + metrics: &SmiteCoordinatorMetrics, +) -> Result { + let smite_proto_action = SmiteProtoAction::decode(message_data).unwrap(); + let action_id = if smite_proto_action.id == 0 { + metrics.action_id_snowflake_generation_proto.incr(); + snowflake_client.generate_id().await? + } else { + smite_proto_action.id + }; + let action_name = smite_proto_action + .data + .unwrap() + .to_string() + .to_case(Case::Snake); + Ok(proto::SmiteCoordinatorAction { + ack_id, + action_id, + action_name, + action_data: Some(ActionData::ProtoActionData(message_data.into())), + secret_data: None, + timestamp: Some(message_timestamp), + }) +} + +async fn decode_msgpack_json_message( + message_data: &[u8], + ack_id: u64, + message_timestamp: Timestamp, + snowflake_client: &SnowflakeClient, + metrics: &SmiteCoordinatorMetrics, +) -> Result { + // This whole function can probably be optimized way better, but in the interest of time I am leaving + // it in a working state for now. + #[derive(Deserialize, Debug)] + struct PubsubAction { + id: Option, + name: String, + data: Value, + secret_data: Option, + } + + let decoded = MsgPack::parse(message_data)?; + let decoded = decoded.as_string()?; + let pubsub_action: PubsubAction = serde_json::from_str(decoded.as_str())?; + + let serde_json_vec = serde_json::to_vec(&pubsub_action.data)?; + let optional_secret_data = match &pubsub_action.secret_data { + Some(secret_data) => Some(SecretData::JsonSecretData(serde_json::to_vec(secret_data)?)), + _ => None, + }; + + // old msgpack parsing + // let mut out = Vec::with_capacity(1024 * 6); + // let mut de = serde_json::Deserializer::from_slice(serde_json_vec.as_slice()); + // let mut se = rmp_serde::Serializer::new(&mut out); + // serde_transcode::transcode(&mut de, &mut se).unwrap(); + + let action_id = match pubsub_action.id { + Some(id) => id.parse::()?, + None => { + metrics.action_id_snowflake_generation_json.incr(); + snowflake_client.generate_id().await? + } + }; + + Ok(proto::SmiteCoordinatorAction { + ack_id, + action_id, + action_name: pubsub_action.name, + action_data: Some(ActionData::JsonActionData(serde_json_vec)), + secret_data: optional_secret_data, + timestamp: Some(message_timestamp), + }) +} + +async fn decrypt_pubsub_message( + kms_envelope: Arc, + message_data: &[u8], +) -> Result> { + kms_envelope + .decrypt(message_data) + .await + .map_err(|err| anyhow!("message decryption failed: {}", err.to_string())) +} + +async fn create_action_from_pubsub_message( + kms_envelope: Arc, + message_data: &[u8], + message_attributes: &HashMap, + ack_id: u64, + message_timestamp: Timestamp, + snowflake_client: &SnowflakeClient, + metrics: &SmiteCoordinatorMetrics, +) -> Result { + let decrypted_message_vector = match message_attributes.get("encrypted") { + Some(is_encrypted) if is_encrypted == "true" => { + Some(decrypt_pubsub_message(kms_envelope, message_data).await?) + } + _ => None, + }; + let message_data = match &decrypted_message_vector { + Some(data) => &data[..], + None => message_data, + }; + match message_attributes.get("encoding") { + Some(encoding) if encoding == "proto" => { + decode_proto_message( + message_data, + ack_id, + message_timestamp, + snowflake_client, + metrics, + ) + .await + } + _ => { + decode_msgpack_json_message( + message_data, + ack_id, + message_timestamp, + snowflake_client, + metrics, + ) + .await + } + } +} + +async fn create_pubsub_subscription_client( +) -> SubscriberClient> { + let emulator_host = std::env::var("PUBSUB_EMULATOR_HOST").ok(); + + let timeout = Duration::from_secs(5); + + if let Some(emulator_host) = emulator_host { + tracing::info!("Creating subscription client to emulator"); + Connection::new_no_auth( + format!("http://{}", emulator_host).try_into().unwrap(), + timeout, + ) + .create_subscriber_client() + } else { + tracing::info!("Creating subscription client to real pubsub"); + let service_account = + std::env::var("SMITE_COORDINATOR_SERVICE_ACCOUNT").unwrap_or("default".to_string()); + let client = GCPMetadataClient::new(service_account).unwrap(); + Connection::from_metadata_client( + client, + timeout, + Duration::from_secs(24000), + GOOGLE_PUBSUB_DOMAIN, + ) + .await + .unwrap() + .create_subscriber_client() + } +} + +pub async fn start_pubsub_subscriber( + snowflake_client: Arc, + priority_queue_sender: PriorityQueueSender, + metrics: Arc, +) -> Result<()> { + let subscriber_client = create_pubsub_subscription_client().await; + let subscription_name = if std::env::var("PUBSUB_EMULATOR_HOST").is_ok() { + PubSubSubscription::new("discord-dev", "rules-sink") + } else { + let project_id = std::env::var("PUBSUB_SUBSCRIPTION_PROJECT_ID") + .unwrap_or("discord-anti-abuse-prd".to_string()); + + let subscription_id = + std::env::var("PUBSUB_SUBSCRIPTION_ID").unwrap_or("smite-coordinator-test".to_string()); + + PubSubSubscription::new(project_id, subscription_id) + }; + + let kek_uri = std::env::var("PUBSUB_ENCRYPTION_KEY_URI").unwrap_or("".to_string()); + + let kms_envelope = Connection::from_metadata_client( + GCPMetadataClient::new("default".into())?, + Duration::from_secs(5), + Duration::from_secs(24000), + GOOGLE_KMS_DOMAIN, + ) + .await? + .create_kms_aes_gcm_envelope(kek_uri, Vec::new(), true)?; + + let kms_envelope = Arc::new(kms_envelope); + let max_messages = std::env::var("PUBSUB_MAX_MESSAGES") + .unwrap_or("5000".to_string()) + .parse::() + .unwrap(); + let max_processing_messages = std::env::var("PUBSUB_MAX_PROCESSING_MESSAGES") + .unwrap_or("5000".to_string()) + .parse::() + .unwrap(); + let max_time_to_send_to_async_queue = TokioDuration::from_millis( + std::env::var("MAX_TIME_TO_SEND_TO_ASYNC_QUEUE_MS") + .unwrap_or("500".to_string()) + .parse::() + .unwrap(), + ); + let max_acking_receiver_wait_time = TokioDuration::from_millis( + std::env::var("MAX_ACKING_RECEIVER_WAIT_TIME_MS") + .unwrap_or("60000".to_string()) + .parse::() + .unwrap(), + ); + + tracing::info!( + {subscription_name = %subscription_name}, + "creating streaming pull manager" + ); + let flow_control = FlowControl::default() + .set_max_messages(max_messages) + .set_max_processing_messages(max_processing_messages) + .set_max_bytes(1024 * 1024 * 1024); + StreamingPullManager::new( + subscriber_client, + subscription_name, + flow_control, + SpawnTaskPerMessageHandler::new(move |message: DetachedMessage| { + let metrics = metrics.clone(); + let message_id = message.message_id.clone(); + let priority_queue_sender = priority_queue_sender.clone(); + let snowflake_client = snowflake_client.clone(); + let kms_envelope = kms_envelope.clone(); + async move { + let message_attributes = message.attributes(); + + let ack_id: u64 = { + let mut rng = rand::thread_rng(); + rng.gen() + }; + + let action = create_action_from_pubsub_message( + kms_envelope, + message.data.as_slice(), + message_attributes, + ack_id, + message.publish_time(), + snowflake_client.as_ref(), + &metrics, + ).await + .map_err(|_| ())?; + let (ackable_action, acking_receiver) = AckableAction::new(action); + + tracing::debug!({ack_id = %ack_id, message_id=%message_id}, "[pubsub] received pubsub message"); + let send_start_time = Instant::now(); + match timeout(max_time_to_send_to_async_queue, priority_queue_sender.send_async(ackable_action)).await { + Ok(Ok(())) => { + tracing::debug!({message_id=%message_id, ack_id=ack_id}, "[pubsub] sent pubsub message to priority queue"); + metrics.async_classification_added_to_queue.incr(); + }, + Ok(Err(e)) => { + tracing::error!({error=%e},"[pubsub] priority queue send error"); + }, + Err(_) => { + tracing::error!({message_id=%message_id}, "[pubsub] sending to priority queue timed out"); + } + }; + metrics.priority_queue_send_time_async.record(send_start_time.elapsed()); + tracing::debug!({message_id=%message_id, ack_id=ack_id},"[pubsub] waiting on ack or nack"); + + let receive_start_time = Instant::now(); + match timeout(max_acking_receiver_wait_time, acking_receiver).await { // 5 Minutes to return + Ok(Ok(ack_or_nack)) => match ack_or_nack { + AckOrNack::Ack(_optional_execution_result) => { + tracing::debug!({message_id=%message_id, ack_id=ack_id},"[pubsub] acking message"); + metrics.async_classification_result_ack.incr(); + metrics.receiver_ack_time_async.record(receive_start_time.elapsed()); + Ok(()) + }, + AckOrNack::Nack => { + tracing::debug!({message_id=%message_id, ack_id=ack_id},"[pubsub] nacking message"); + metrics.async_classification_result_nack.incr(); + metrics.receiver_ack_time_async.record(receive_start_time.elapsed()); + Err(()) + }, + }, + Ok(Err(recv_error)) => { + tracing::error!({message_id=%message_id, recv_error=%recv_error, ack_id=ack_id},"[pubsub] acking sender dropped"); + metrics.receiver_ack_time_async.record(receive_start_time.elapsed()); + Err(()) + }, + Err(_) => { + tracing::error!({message_id=%message_id, ack_id=ack_id}, "[pubsub] waiting for ack/nack timed out"); + metrics.receiver_ack_time_async.record(receive_start_time.elapsed()); + Err(()) + }, + } + } + }), + MetricsClientBuilder::new("smite_coordinator.pull"), + ) + .gracefully_stop_on_signal(exit_signal(), Duration::from_secs(30)) + .await; + Result::Ok(()) +} + +// TODO: Fix these tests + +// #[cfg(test)] +// mod tests { +// use std::{collections::HashMap, fs::File, io::Read}; + +// use msgpack_simple::MsgPack; +// use prost::Message; +// use prost_types::Timestamp; +// // use protobuf_json_mapping; +// use serde_json::json; + +// // use discord_smite_rpc_actions_proto::SmiteCoordinatorAction as SmiteRpcAction; +// use crate::proto; +// use std::io::Cursor; +// use std::str; + +// use super::create_action_from_pubsub_message; + +// // #[test] +// // fn test_create_action_from_pubsub_message_1() { +// // let action_json = json!({ +// // "id": "123456789", +// // "name": "guild_invite_created", +// // "data": { +// // "char": "abc", +// // "int": 1i64, +// // "float2": 1.1_f64 +// // }, + +// // }); +// // let encoded = MsgPack::String(action_json.to_string()).encode(); + +// // let prost_action = +// // create_action_from_pubsub_message(encoded.as_slice(), 12344242, Timestamp::default()); +// // println!("{:?}", prost_action); +// // assert!(prost_action.is_ok(), "prost action decoding failed"); +// // let prost_action = prost_action.unwrap(); +// // let data = MsgPack::parse(&prost_action.action_data).unwrap(); +// // println!("{:?}", data); +// // let mut data: HashMap = data +// // .as_map() +// // .unwrap() +// // .into_iter() +// // .map(|v| (v.key.as_string().unwrap(), v.value)) +// // .collect(); + +// // let x = data.remove("char").unwrap().as_string().unwrap(); + +// // assert_eq!(x, "abc".to_string()); +// // } + +// // #[test] +// // fn test_create_action_from_pubsub_message_2() { +// // let action_json = json!({ +// // "id": "123456789", +// // "name": "guild_invite_created", +// // "data": { +// // "char": "abc", +// // "int": 1i64, +// // "float2": 1.1_f64 +// // }, + +// // }); +// // // let encoded = MsgPack::String(action_json.to_string()).encode(); + +// // let prost_action = create_action_from_pubsub_message( +// // action_json.to_string().bytes().collect(), +// // 12344242, +// // Timestamp::default(), +// // ); +// // println!("{:?}", prost_action); +// // assert!(prost_action.is_ok(), "prost action decoding failed"); +// // let prost_action = prost_action.unwrap(); +// // let data = MsgPack::parse(&prost_action.action_data).unwrap(); +// // println!("{:?}", data); +// // let mut data: HashMap = data +// // .as_map() +// // .unwrap() +// // .into_iter() +// // .map(|v| (v.key.as_string().unwrap(), v.value)) +// // .collect(); + +// // let x = data.remove("char").unwrap().as_string().unwrap(); + +// // assert_eq!(x, "abc".to_string()); +// // } +// #[test] +// fn test_create_action_from_pubsub_proto_action() { +// let mut file = File::open("pubsub_messages.json").unwrap(); +// let mut data = String::new(); +// file.read_to_string(&mut data).unwrap(); +// let json: serde_json::Value = serde_json::from_str(&data).unwrap(); +// let action_jsons = json.as_array().expect("was not array"); +// let action_json = action_jsons[0].as_object().expect("is not map"); +// println!("{:?}", action_json); +// let action_data = action_json +// .get("message") +// .unwrap() +// .as_object() +// .unwrap() +// .get("data") +// .unwrap() +// .as_str() +// .unwrap(); + +// println!("{:?}", action_data); +// let action_data = str::from_utf8(action_data.as_bytes()).unwrap(); +// println!("{:?}", action_data); +// // let action_data = base64::decode(action_data).unwrap(); + +// // let mut action_prost: SmiteRpcAction = +// // SmiteRpcAction::decode(&mut Cursor::new(action_data.to_string())).unwrap(); +// println!("--------"); +// let test_object_data = "CWUQhNrjpmQOGsgBCgsJCwCCONT0dQYQARK4AQogfnvDH9y83BpR5FraKYJSCxTDrQkSb1axRgl9i82pYwgQARoLCI2fiZYGEPeX1CciCwjksNCbBhC8i4FgKgsI5MaGmwYQvIuBYDItCg0KCzk1LjIuMTIuMTY2EgkKB0FuZHJvaWQaEQoPRGlzY29yZCBBbmRyb2lkOi8KDwoNMTc2LjIxOS40Mi4zNBIJCgdBbmRyb2lkGhEKD0Rpc2NvcmQgQW5kcm9pZEILCPqihpsGEOj7p0c="; +// let decoded_base64 = base64::decode(test_object_data).unwrap(); +// let mut test_action = proto::SmiteCoordinatorAction::default(); +// test_action.id = 20; + +// println!("{:?}", test_action); +// let x = test_action.encode_to_vec(); +// println!("{:?}", x); +// println!("--------"); +// println!("{:?}", decoded_base64); +// let mut action_prost: SmiteRpcAction = +// SmiteRpcAction::decode(decoded_base64.as_slice()).unwrap(); +// println!("{:?}", action_prost); + +// let output = protobuf_json_mapping::print_to_string(action_prost); +// println!("{:?}", output); + +// // let prost_action = create_action_from_pubsub_message( +// // action_json.to_string().bytes().collect(), +// // 12344242, +// // Timestamp::default(), +// // ); +// // println!("{:?}", prost_action); +// // assert!(prost_action.is_ok(), "prost action decoding failed"); +// // let prost_action = prost_action.unwrap(); +// // let data = MsgPack::parse(&prost_action.action_data).unwrap(); +// // println!("{:?}", data); +// // let mut data: HashMap = data +// // .as_map() +// // .unwrap() +// // .into_iter() +// // .map(|v| (v.key.as_string().unwrap(), v.value)) +// // .collect(); + +// // let x = data.remove("char").unwrap().as_string().unwrap(); + +// // assert_eq!(x, "abc".to_string()); +// } +// } diff --git a/osprey_coordinator/src/shutdown_handler.rs b/osprey_coordinator/src/shutdown_handler.rs new file mode 100644 index 0000000..9ad7b09 --- /dev/null +++ b/osprey_coordinator/src/shutdown_handler.rs @@ -0,0 +1,18 @@ +use crate::priority_queue::{PriorityQueueReceiver, PriorityQueueSender}; +use crate::signals; + +pub fn spawn_shutdown_handler( + priority_queue_sender: PriorityQueueSender, + priority_queue_receiver: PriorityQueueReceiver, +) { + tokio::spawn(async move { + tracing::info!("shutdown handler spawned - waiting on exit signal"); + signals::exit_signal().await; + tracing::info!("got exit signal"); + priority_queue_receiver.nack_all_async(); + tracing::info!("nacked all outstanding async actions"); + tokio::time::sleep(tokio::time::Duration::from_secs(15)).await; + priority_queue_sender.close(); + tracing::info!("closed priority queue"); + }); +} diff --git a/osprey_coordinator/src/signals/mod.rs b/osprey_coordinator/src/signals/mod.rs new file mode 100644 index 0000000..9af2bff --- /dev/null +++ b/osprey_coordinator/src/signals/mod.rs @@ -0,0 +1,27 @@ +/// Returns a future which will resolve when Ctrl-C is received. +/// +/// Useful to know when the process should begin its cleanup and graceful shutdown. +#[cfg(windows)] +pub async fn exit_signal() { + tokio::signal::ctrl_c().await; +} + +/// Returns a future which will resolve when SIGINT/SIGTERM are sent to the process. +/// +/// Useful to know when the process should begin its cleanup and graceful shutdown. +#[cfg(unix)] +pub async fn exit_signal() { + use tokio::signal::unix::{Signal, SignalKind}; + + let mut term = create_signal(SignalKind::terminate()); + let mut ctrl_c = create_signal(SignalKind::interrupt()); + + tokio::select! { + _ = term.recv() => {}, + _ = ctrl_c.recv() => {}, + } + + fn create_signal(kind: SignalKind) -> Signal { + tokio::signal::unix::signal(kind).expect("couldn't create signal.") + } +} diff --git a/osprey_coordinator/src/smite_bidirectional_stream.rs b/osprey_coordinator/src/smite_bidirectional_stream.rs new file mode 100644 index 0000000..0dab867 --- /dev/null +++ b/osprey_coordinator/src/smite_bidirectional_stream.rs @@ -0,0 +1,432 @@ +use crate::coordinator_metrics::SmiteCoordinatorMetrics; +use crate::priority_queue::ActionAcker; +use crate::priority_queue::{PriorityQueueReceiver, PriorityQueueSender}; +use crate::proto; +use anyhow::{anyhow, Context, Result}; +use proto::action_request::ActionRequest; +use std::sync::Arc; +use std::{error::Error, io::ErrorKind}; +use tokio::sync::mpsc::{self, Sender}; +use tokio::time::{timeout, Duration, Instant}; +use tokio_stream::{wrappers::ReceiverStream, StreamExt}; + +use crate::metrics::counters::StaticCounter; +use crate::metrics::histograms::StaticHistogram; + +fn match_for_io_error(err_status: &tonic::Status) -> Option<&std::io::Error> { + let mut err: &(dyn Error + 'static) = err_status; + + loop { + if let Some(io_err) = err.downcast_ref::() { + return Some(io_err); + } + + // h2::Error do not expose std::io::Error with `source()` + // https://github.com/hyperium/h2/pull/462 + if let Some(h2_err) = err.downcast_ref::() { + if let Some(io_err) = h2_err.get_io() { + return Some(io_err); + } + } + + err = match err.source() { + Some(err) => err, + None => return None, + }; + } +} + +#[derive(Debug)] +struct OutstandingActionState { + action_acker: ActionAcker, + send_time: Instant, + client_details: proto::ClientDetails, +} + +#[derive(Debug)] +enum ClientState { + NoOutstandingAction, + OutstandingAction(OutstandingActionState), +} + +pub struct SmiteCoordinatorServer { + priority_queue_receiver: PriorityQueueReceiver, + #[allow(unused)] + priority_queue_sender: PriorityQueueSender, // TODO: use this for retrying sync actions + metrics: Arc, +} + +impl SmiteCoordinatorServer { + pub fn new( + priority_queue_sender: PriorityQueueSender, + priority_queue_receiver: PriorityQueueReceiver, + metrics: Arc, + ) -> SmiteCoordinatorServer { + SmiteCoordinatorServer { + priority_queue_sender, + priority_queue_receiver, + metrics, + } + } +} + +fn handle_action_request( + action_request: ActionRequest, + current_client_state: ClientState, + metrics: Arc, +) -> Result { + match (action_request, current_client_state) { + (ActionRequest::Initial(client_details), ClientState::NoOutstandingAction) => { + Ok(client_details) + } + (ActionRequest::Initial(_), ClientState::OutstandingAction(_)) => Err(anyhow!( + "got an initial action request while there was an outstanding action" + )), + (ActionRequest::AckOrNack(ack_or_nack), ClientState::NoOutstandingAction) => Err(anyhow!( + "got an {:?} with no outstanding action", + ack_or_nack + )), + (ActionRequest::AckOrNack(ack_or_nack), ClientState::OutstandingAction(state)) => { + let duration = Instant::now().duration_since(state.send_time); + metrics.action_outstanding_duration.record(duration); + state.action_acker.ack_or_nack( + ack_or_nack + .ack_or_nack + .context("no `ack_or_nack` in proto")?, + ); + Ok(state.client_details) + } + } +} + +enum UpdateClientStateOrDisconnect { + UpdateClientState(ClientState), + ClientRequestedDisconnect, + #[allow(dead_code)] + ActionReceiverClosedDisconnect, + ActionReceiverTimedOut, +} + +async fn handle_request( + client_state: ClientState, + sender: &Sender>, + request: proto::Request, + action_receiver: &PriorityQueueReceiver, + metrics: Arc, + receive_timeout: Duration, +) -> Result { + match request + .request + .context("request object missing from proto")? + { + proto::request::Request::ActionRequest(action_request) => { + let action_request = action_request + .action_request + .context("no `action_request.action_request` in `ActionRequest` proto")?; + let client_details = + handle_action_request(action_request, client_state, metrics.clone())?; + tracing::debug!("awaiting action from priority queue"); + let priority_queue_receive_start_time = Instant::now(); + let result = timeout(receive_timeout, action_receiver.recv(metrics.clone())).await; + let ackable_action = match result { + Ok(Ok(ackable_action)) => ackable_action, + Ok(Err(_)) | Err(_) => { + tracing::error!( + "Took too long to get action from priority queue, disconnecting" + ); + metrics + .priority_queue_receive_time + .record(Instant::now().duration_since(priority_queue_receive_start_time)); + return Ok(UpdateClientStateOrDisconnect::ActionReceiverTimedOut); + } + }; + metrics + .priority_queue_receive_time + .record(Instant::now().duration_since(priority_queue_receive_start_time)); + let (action, action_acker) = ackable_action.into_action(); + sender.send(Ok(action)).await?; + Ok(UpdateClientStateOrDisconnect::UpdateClientState( + ClientState::OutstandingAction(OutstandingActionState { + action_acker, + send_time: Instant::now(), + client_details, + }), + )) + } + proto::request::Request::Disconnect(disconnect) => { + let ack_or_nack = disconnect + .ack_or_nack + .context("no `ack_or_nack` in `disconnect` proto")? + .ack_or_nack + .context("no `ack_or_nack.ack_or_nack` in `disconnect` proto")?; + if let ClientState::OutstandingAction(state) = client_state { + state.action_acker.ack_or_nack(ack_or_nack); + } + Ok(UpdateClientStateOrDisconnect::ClientRequestedDisconnect) + } + } +} + +#[tonic::async_trait] +impl proto::smite_coordinator_service_server::SmiteCoordinatorService for SmiteCoordinatorServer { + type SmiteBidirectionalStreamStream = + ReceiverStream>; + + async fn smite_bidirectional_stream( + &self, + request: tonic::Request>, + ) -> Result, tonic::Status> { + tracing::debug!( + { connection =? request.metadata() }, + "New Connection Received" + ); + let mut in_stream = request.into_inner(); + self.metrics.new_connection_established.incr(); + let (tx, rx) = mpsc::channel(128); + let action_receiver = self.priority_queue_receiver.clone(); + let metrics = self.metrics.clone(); + let max_pq_receive_await_time_ms = Duration::from_millis( + std::env::var("MAX_PQ_RECEIVE_AWAIT_TIME_MS") + .unwrap_or("5000".to_string()) + .parse::() + .unwrap(), + ); + tokio::spawn(async move { + let mut client_state = ClientState::NoOutstandingAction {}; + + // TODO: refactor the code to honor the invariants of: the first request should always be an + // InitialActionRequest and every request after that will either be an AckingRequest or AckingDisconnect + // let initial_request = in_stream.next().await.unwrap().unwrap(); + // let client_details = match initial_request.request.expect("request must exist") { + // proto::request::Request::ActionRequest(action_request) => { + // match action_request.action_request.expect("must exist") { + // ActionRequest::Initial(client_details) => client_details, + // ActionRequest::AckOrNack(_) => unreachable!(), + // } + // } + // proto::request::Request::Disconnect(_) => unreachable!(), + // }; + + while let Some(result) = in_stream.next().await { + match result { + Ok(request) => { + tracing::debug!({request=?request},"got request"); + client_state = match handle_request( + client_state, + &tx, + request, + &action_receiver, + metrics.clone(), + max_pq_receive_await_time_ms, + ) + .await + { + Ok(directive) => match directive { + UpdateClientStateOrDisconnect::UpdateClientState( + new_client_state, + ) => new_client_state, + UpdateClientStateOrDisconnect::ClientRequestedDisconnect => { + tracing::debug!("client requested a disconnect"); + metrics.client_disconnected_gracefully.incr(); + break; + } + UpdateClientStateOrDisconnect::ActionReceiverClosedDisconnect => { + tracing::debug!("disconnecting client because receiver closed"); + metrics.client_disconnected_receiver_closed.incr(); + break; + } + UpdateClientStateOrDisconnect::ActionReceiverTimedOut => { + tracing::debug!( + "disconnecting client because receiver timed out" + ); + metrics.client_disconnected_receiver_timeout.incr(); + break; + } + }, + Err(error) => { + tracing::error!({error=%error},"error in stream"); + metrics.client_disconnected_stream_error.incr(); + // commenting this out for now + // we might not have to send an aborted when we get an error + // tx.send(Err(tonic::Status::new( + // tonic::Code::Aborted, + // error.to_string(), + // ))) + // .await + // .expect("output stream must be open"); + break; + } + } + } + Err(err) => { + if let Some(io_err) = match_for_io_error(&err) { + if io_err.kind() == ErrorKind::BrokenPipe { + tracing::error!("client disconnected: broken pipe"); + metrics.client_disconnected_broken_pipe.incr(); + break; + } + } + + match tx.send(Err(err)).await { + Ok(_) => (), + Err(_err) => break, // response was dropped + } + } + } + } + + tracing::debug!("stream ended"); + }); + + let out_stream = ReceiverStream::new(rx); + Ok(tonic::Response::new(out_stream)) + } +} + +#[cfg(test)] +mod tests { + + use crate::coordinator_metrics::SmiteCoordinatorMetrics; + use crate::metrics::emit_worker::SpawnEmitWorker; + use crate::metrics::new_client; + use crate::proto::smite_coordinator_action::ActionData; + use crate::proto::smite_coordinator_action::SecretData; + use proto::smite_coordinator_service_server::SmiteCoordinatorService; + + use crate::priority_queue::create_ackable_action_priority_queue; + use crate::priority_queue::AckableAction; + + use super::*; + + #[tokio::test] + async fn golden_path_bidirection_streaming_test() -> Result<()> { + // Simple golden path test that adds two actions to the queue and asserts that a properly + // formed bidirectional streaming request is returned the actions in that order + + tracing_subscriber::fmt::init(); + let (priority_queue_sender, priority_queue_receiver) = + create_ackable_action_priority_queue(); + let metrics = SmiteCoordinatorMetrics::new(); + let _worker_guard = metrics + .clone() + .spawn_emit_worker(new_client("smite_coordinator").unwrap()); + + let ackable_action = proto::SmiteCoordinatorAction { + ack_id: 1, + action_id: 1, + action_name: "test_action".into(), + timestamp: None, + action_data: Some(ActionData::JsonActionData( + "{\"action\": \"test action data 1\"}".into(), + )), + secret_data: Some(SecretData::JsonSecretData( + "{\"secret\": \"test secret data 1\"}".into(), + )), + }; + let (ackable_action, _receiver_drop_guard_1) = AckableAction::new(ackable_action); + priority_queue_sender + .send_sync(ackable_action) + .await + .unwrap(); + + let ackable_action_2 = proto::SmiteCoordinatorAction { + ack_id: 2, + action_id: 2, + action_name: "test_action".into(), + timestamp: None, + action_data: Some(ActionData::JsonActionData( + "{\"action\": \"test action data 2\"}".into(), + )), + secret_data: Some(SecretData::JsonSecretData( + "{\"secret\": \"test secret data 2\"}".into(), + )), + }; + let (ackable_action, _receiver_drop_guard_2) = AckableAction::new(ackable_action_2); + priority_queue_sender + .send_sync(ackable_action) + .await + .unwrap(); + + let server = SmiteCoordinatorServer::new( + priority_queue_sender.clone(), + priority_queue_receiver, + metrics.clone(), + ); + + let initial_action_request = proto::Request { + request: Some(proto::request::Request::ActionRequest( + proto::ActionRequest { + action_request: Some(proto::action_request::ActionRequest::Initial( + proto::ClientDetails::default(), + )), + }, + )), + }; + + let acking_action_request = proto::Request { + request: Some(proto::request::Request::ActionRequest( + proto::ActionRequest { + action_request: Some(proto::action_request::ActionRequest::AckOrNack( + proto::AckOrNack { + ack_id: 0, + ack_or_nack: Some(proto::ack_or_nack::AckOrNack::Ack(proto::Ack { + execution_result: None, + verdicts: None, + })), + }, + )), + }, + )), + }; + + let req = crate::tonic_mock::streaming_request(vec![ + initial_action_request.clone(), + acking_action_request.clone(), + ]); + + let res = server + .smite_bidirectional_stream(req) + .await + .expect("error in stream"); + + println!("finish connection"); + + let mut result = Vec::new(); + let mut messages = res.into_inner(); + while let Some(v) = messages.next().await { + println!("got message: {:?}", &v); + result.push(v.expect("error from stream")) + } + + print!("{:?}", result); + + assert_eq!(result[0].action_id, 1); + assert_eq!(result[1].action_id, 2); + assert_eq!( + result[0].action_data, + Some(ActionData::JsonActionData( + "{\"action\": \"test action data 1\"}".into() + )) + ); + assert_eq!( + result[1].action_data, + Some(ActionData::JsonActionData( + "{\"action\": \"test action data 2\"}".into() + )) + ); + assert_eq!( + result[0].secret_data, + Some(SecretData::JsonSecretData( + "{\"secret\": \"test secret data 1\"}".into() + )) + ); + assert_eq!( + result[1].secret_data, + Some(SecretData::JsonSecretData( + "{\"secret\": \"test secret data 2\"}".into() + )) + ); + + Ok(()) + } +} diff --git a/osprey_coordinator/src/snowflake_client.rs b/osprey_coordinator/src/snowflake_client.rs new file mode 100644 index 0000000..174c161 --- /dev/null +++ b/osprey_coordinator/src/snowflake_client.rs @@ -0,0 +1,104 @@ +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Debug, Serialize, Deserialize)] +struct SnowflakeRequest { + count: u32, +} + +#[derive(Error, Debug)] +pub enum SnowflakeClientError { + #[error(transparent)] + IdGenerateError(#[from] reqwest::Error), + + #[error("no snowflake id returned from snowflake service")] + NoIdGeneratedError, +} + +pub struct SnowflakeClient { + snowflake_api_endpoint: String, + reqwest_client: reqwest::Client, + // buffer: SnowflakeBuffer, + // request_size: u32, +} + +impl SnowflakeClient { + pub fn new(snowflake_api_endpoint: String) -> Self { + Self { + snowflake_api_endpoint, + reqwest_client: reqwest::Client::new(), + // note: to get a buffer working without hindering performance, we'd need to figure + // out a way to pass around the buffer without the dreaded Arc>. because i dont wanna + // figure that out, and it'd be definitely overkill for now, i'm just gonna Arc<> and make the + // client only perform single id requests. + // + // buffer: SnowflakeBuffer::new(), + // // todo: it would be cool if this was calculated based on a rolling window + // // of the number of ids requested in the last 500ms or so . . but that's way + // // too overengineered for now lol + // request_size: 1, + } + } + + pub async fn generate_id(&self) -> Result { + // osprey-snowflake api spec: https://github.com/ayubun/snowflake-id-worker?tab=readme-ov-file#api-spec + let mut snowflake_response: Vec = self + .reqwest_client + .post(format!("{}/generate", self.snowflake_api_endpoint)) + .json(&SnowflakeRequest { count: 1 }) + .send() + .await? + .json() + .await?; + + let id = snowflake_response + .pop() + .ok_or(SnowflakeClientError::NoIdGeneratedError)?; + + Ok(id) + } +} + +// struct SnowflakeBuffer { +// buffer: Vec, +// last_buffer_fill: SystemTime, +// buffer_ttl: Duration, +// } + +// impl SnowflakeBuffer { +// pub fn new() -> Self { +// Self { +// buffer: Vec::new(), +// last_buffer_fill: UNIX_EPOCH, +// // we don't want to use buffered ids older than 1 millisecond, since that +// // is the precision of snowflake timestamps, and we want our snowflakes +// // to be as close to the time of generation as possible. +// buffer_ttl: Duration::from_millis(1), +// } +// } + +// /// returns the next id in the buffer, or `None` if the buffer +// /// cannot provide an id (either empty or buffer is too old) +// pub fn next_id(&mut self) -> Option { +// if !self.buffer.is_empty() && self +// .last_buffer_fill +// .elapsed() +// .unwrap_or_else(|_| { +// UNIX_EPOCH +// .elapsed() +// .expect("invariant: literally impossible") +// }) +// .as_nanos() +// > self.buffer_ttl.as_nanos() +// { +// self.buffer.clear(); +// } +// self.buffer.pop() +// } + +// /// fills the buffer with the given ids. +// pub fn fill(&mut self, ids: Vec) { +// self.buffer = ids; +// self.last_buffer_fill = SystemTime::now(); +// } +// } diff --git a/osprey_coordinator/src/sync_action_rpc.rs b/osprey_coordinator/src/sync_action_rpc.rs new file mode 100644 index 0000000..809b410 --- /dev/null +++ b/osprey_coordinator/src/sync_action_rpc.rs @@ -0,0 +1,204 @@ +use crate::metrics::counters::StaticCounter; +use crate::metrics::histograms::StaticHistogram; +use crate::snowflake_client::SnowflakeClient; +use crate::{ + coordinator_metrics::SmiteCoordinatorMetrics, + label_service_client::LabelServiceClient, + priority_queue::AckableAction, + priority_queue::{AckOrNack, PriorityQueueSender}, + proto::{self, smite_coordinator_sync_action}, +}; +use anyhow::{anyhow, Context, Result}; +use std::sync::Arc; +use tokio::time::Instant; + +use rand::Rng; +use smite_coordinator_sync_action::smite_coordinator_sync_action_service_server::SmiteCoordinatorSyncActionService; +use smite_coordinator_sync_action::ProcessActionRequest; + +pub(crate) struct SyncActionServer { + snowflake_client: Arc, + priority_queue_sender: PriorityQueueSender, + metrics: Arc, + label_service_client: LabelServiceClient, +} + +impl SyncActionServer { + pub fn new( + snowflake_client: Arc, + priority_queue_sender: PriorityQueueSender, + metrics: Arc, + label_service_client: LabelServiceClient, + ) -> SyncActionServer { + SyncActionServer { + snowflake_client, + priority_queue_sender, + metrics, + label_service_client, + } + } +} + +async fn create_smite_coordinator_action( + ack_id: u64, + action_request: &smite_coordinator_sync_action::ProcessActionRequest, + snowflake_client: &SnowflakeClient, +) -> Result<(proto::SmiteCoordinatorAction, Vec)> { + // generate snowflake if one is not provided, to match the behaviour in pubsub.rs + let action_id = match action_request.action_id { + Some(id) => match id { + // handle 0 as none-type, since protos default u64 to 0 + 0 => snowflake_client.generate_id().await?, + _ => id, + }, + None => snowflake_client.generate_id().await?, + }; + if action_request.action_name.is_empty() { + return Err(anyhow!("`action_name` must not be empty")); + } + let smite_coordinator_action = proto::SmiteCoordinatorAction { + ack_id, + action_id, + action_name: action_request.action_name.clone(), + action_data: Some(proto::smite_coordinator_action::ActionData::JsonActionData( + action_request.action_data_json.clone().into(), + )), + secret_data: None, + timestamp: Some( + action_request + .timestamp + .as_ref() + .context("`timestamp` not found")? + .clone(), + ), + }; + + Ok(( + smite_coordinator_action, + action_request.requested_entities.clone(), + )) +} + +impl SyncActionServer { + async fn try_process_action( + &self, + ack_id: u64, + action_request: &ProcessActionRequest, + ) -> Result, tonic::Status> + { + let unvalidated_action_id = action_request.action_id; + + let (smite_coordinator_action, requested_entities) = match create_smite_coordinator_action( + ack_id, + action_request, + self.snowflake_client.as_ref(), + ) + .await + { + Ok(result) => result, + Err(error) => { + tracing::error!({error=%error, ack_id=ack_id, action_id=unvalidated_action_id},"[rpc] deserialization error"); + self.metrics + .sync_classification_failure_deserialization + .incr(); + return Err(tonic::Status::new(tonic::Code::Aborted, error.to_string())); + } + }; + + let action_id = smite_coordinator_action.action_id; + + let (ackable_action, acking_receiver) = AckableAction::new(smite_coordinator_action); + + let send_start_time = Instant::now(); + match self.priority_queue_sender.send_sync(ackable_action).await { + Ok(_) => { + tracing::debug!({action_id=%action_id, ack_id=ack_id}, "[rpc] sent message to priority queue") + } + Err(e) => { + self.metrics.sync_classification_failure_pq_send.incr(); + tracing::error!({error=%e, action_id=%action_id, ack_id=ack_id},"[rpc] tried to send action to closed channel"); + return Err(tonic::Status::new(tonic::Code::Unavailable, e.to_string())); + } + }; + self.metrics + .priority_queue_send_time_sync + .record(send_start_time.elapsed()); + tracing::debug!({action_id=%action_id, ack_id=ack_id},"[rpc] waiting on ack or nack"); + + let receive_start_time = Instant::now(); + match acking_receiver.await { + Ok(ack_or_nack) => match ack_or_nack { + AckOrNack::Ack(verdicts) => { + tracing::debug!({action_id=%action_id, ack_id=ack_id},"[rpc] acking message"); + let entities = match self + .label_service_client + .get_entities(requested_entities) + .await + { + Ok(entities) => entities, + Err(error) => { + tracing::error!({error=%error},"label service call failed"); + self.metrics + .sync_classification_failure_label_service + .incr(); + return Err(tonic::Status::aborted("label service call failed")); + } + }; + + let response = + smite_coordinator_sync_action::ProcessActionResponse { entities, verdicts }; + + self.metrics.sync_classification_result_ack.incr(); + self.metrics + .receiver_ack_time_sync + .record(receive_start_time.elapsed()); + Ok(tonic::Response::new(response)) + } + AckOrNack::Nack => { + tracing::debug!({action_id=%action_id, ack_id=ack_id},"[rpc] nacking message"); + self.metrics.sync_classification_result_nack.incr(); + self.metrics + .receiver_ack_time_sync + .record(receive_start_time.elapsed()); + Err(tonic::Status::aborted("action nacked")) + } + }, + Err(recv_error) => { + tracing::error!({action_id=%action_id, recv_error=%recv_error, ack_id=ack_id},"[rpc] acking sender dropped"); + self.metrics + .sync_classification_failure_oneshot_dropped + .incr(); + self.metrics + .receiver_ack_time_sync + .record(receive_start_time.elapsed()); + Err(tonic::Status::internal("acking onshot dropped")) + } + } + } +} + +#[tonic::async_trait] +impl SmiteCoordinatorSyncActionService for SyncActionServer { + async fn process_action( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> + { + self.metrics.sync_classification_action_received.incr(); + let action_request = request.into_inner(); + tracing::debug!({action_request=?action_request}, "[rpc] action request received"); + + let ack_id: u64 = { + let mut rng = rand::thread_rng(); + rng.gen() + }; + + match self.try_process_action(ack_id, &action_request).await { + response @ Ok(_) => response, + Err(e) => { + tracing::error!("initial process_action attempt failed, retrying: {}", e); + self.try_process_action(ack_id, &action_request).await + } + } + } +} diff --git a/osprey_coordinator/src/tokio_utils/mod.rs b/osprey_coordinator/src/tokio_utils/mod.rs new file mode 100644 index 0000000..fbe2195 --- /dev/null +++ b/osprey_coordinator/src/tokio_utils/mod.rs @@ -0,0 +1,65 @@ +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll}, +}; + +use tokio::task::{JoinError, JoinHandle}; + +mod unbounded_receiver_chunker; + +pub use unbounded_receiver_chunker::{ + ChunkSizeConfig as UnboundedReceiverChunkSizeConfig, UnboundedReceiverChunker, +}; + +/// A helper struct that will invoke [`JoinHandle::abort`] when dropped. +#[derive(Debug)] +pub struct AbortOnDrop { + join_handle: Option>, +} + +impl AbortOnDrop { + pub fn new(join_handle: JoinHandle) -> Self { + Self { + join_handle: Some(join_handle), + } + } + + /// Disarms the [`AbortOnDrop`], causing it to no longer abort the join handle when dropped. + pub fn run_forever(mut self) { + self.join_handle = None; + } + + pub fn into_inner(mut self) -> JoinHandle { + self.join_handle + .take() + .expect("invariant: the inner join handle should always be present") + } +} + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + if let Some(join_handle) = self.join_handle.take() { + join_handle.abort(); + } + } +} + +impl From> for AbortOnDrop { + fn from(join_handle: JoinHandle) -> Self { + Self::new(join_handle) + } +} + +impl Future for AbortOnDrop { + type Output = Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let join_handle = self + .join_handle + .as_mut() + .expect("polling after `AbortOnDrop` already completed"); + + Pin::new(join_handle).poll(cx) + } +} diff --git a/osprey_coordinator/src/tokio_utils/unbounded_receiver_chunker.rs b/osprey_coordinator/src/tokio_utils/unbounded_receiver_chunker.rs new file mode 100644 index 0000000..ef684e7 --- /dev/null +++ b/osprey_coordinator/src/tokio_utils/unbounded_receiver_chunker.rs @@ -0,0 +1,136 @@ +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +use tokio::{sync::broadcast::Receiver, sync::mpsc::UnboundedReceiver, time::Instant}; + +use crate::tokio_utils::AbortOnDrop; + +pub struct UnboundedReceiverChunker { + chunker: Chunker, + receiver: UnboundedReceiver, + chunk_timeout: Arc>, + inactive_timeout: Duration, + timeout_at: Instant, + _disconfig_update_guard: Option>>, +} + +/// Trait to get chunk timeout configuration +pub trait ChunkSizeConfig: Send + Sync + 'static { + // Returns the configured value for chunk timeout in seconds + fn chunk_timeout_secs(&self) -> u64; + + // Returns a receiver to subscribe to updates to the configuration + fn subscribe(&self) -> Receiver<()>; +} + +impl UnboundedReceiverChunker { + pub fn new( + receiver: UnboundedReceiver, + max_chunk_size: usize, + default_chunk_timeout: Duration, + inactive_timeout: Duration, + ) -> Self { + Self { + chunker: Chunker::new(max_chunk_size), + chunk_timeout: Arc::new(default_chunk_timeout.into()), + inactive_timeout, + receiver, + timeout_at: Instant::now() + inactive_timeout, + _disconfig_update_guard: None, + } + } + + pub fn with_configurable_timeout(&mut self, disconfig: Arc) { + let chunk_timeout_secs = disconfig.chunk_timeout_secs(); + if chunk_timeout_secs != 0 { + *self.chunk_timeout.write().unwrap() = Duration::from_secs(chunk_timeout_secs); + } + + let mut update_rx = disconfig.subscribe(); + let cloned_chunk_timeout = self.chunk_timeout.clone(); + + let join_handle = tokio::spawn(async move { + while let Ok(_) = update_rx.recv().await { + let chunk_timeout_secs = disconfig.chunk_timeout_secs(); + if chunk_timeout_secs != 0 { + *cloned_chunk_timeout.write().unwrap() = + Duration::from_secs(chunk_timeout_secs); + } + } + }); + self._disconfig_update_guard = Some(Arc::new(join_handle.into())); + } + + pub async fn next_chunk(&mut self) -> Option> { + loop { + match tokio::time::timeout_at(self.timeout_at, self.receiver.recv()).await { + Ok(Some(item)) => { + // We got an item that made us non-empty, adjust the timeout accordingly. + if self.chunker.is_empty() { + self.timeout_at = Instant::now() + *self.chunk_timeout.read().unwrap(); + } + self.chunker.push(item); + } + // The channel will no longer receive any more messages. Flush any chunk we might have, + // then the next time this function is called, we'll go ahead and return None. + Ok(None) => { + return (!self.chunker.is_empty()).then(|| self.chunker.take_current_chunk()) + } + // We timed out, and we have an empty chunker, which means we're going to shut down due to inactivity. + Err(_) if self.chunker.is_empty() => self.receiver.close(), + Err(_) => { + // Our next timeout will cause us to go inactive, unless we receive a message between now and then. + self.timeout_at = Instant::now() + self.inactive_timeout; + return Some(self.chunker.take_current_chunk()); + } + } + + // Check if a chunk has filled, and return it. + if let Some(full_chunk) = self.chunker.take_full_chunk() { + return Some(full_chunk); + } + } + } +} + +struct Chunker { + current_chunk: Vec, + max_chunk_size: usize, + full_chunk: Option>, +} + +impl Chunker { + fn new(max_chunk_size: usize) -> Self { + assert!(max_chunk_size > 0); + + Self { + current_chunk: Vec::with_capacity(max_chunk_size), + full_chunk: None, + max_chunk_size, + } + } + + fn is_empty(&self) -> bool { + self.current_chunk.is_empty() && self.full_chunk.is_none() + } + + fn take_full_chunk(&mut self) -> Option> { + self.full_chunk.take() + } + + fn push(&mut self, item: T) { + self.current_chunk.push(item); + + if self.current_chunk.len() == self.max_chunk_size { + let chunk = self.take_current_chunk(); + self.full_chunk = Some(chunk); + } + } + + fn take_current_chunk(&mut self) -> Vec { + std::mem::replace( + &mut self.current_chunk, + Vec::with_capacity(self.max_chunk_size), + ) + } +} diff --git a/osprey_coordinator/src/tonic_mock/mock.rs b/osprey_coordinator/src/tonic_mock/mock.rs new file mode 100644 index 0000000..2064c37 --- /dev/null +++ b/osprey_coordinator/src/tonic_mock/mock.rs @@ -0,0 +1,104 @@ +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use http_body::Body; +use prost::Message; +use std::{ + collections::VecDeque, + marker::PhantomData, + pin::Pin, + task::{Context, Poll}, +}; + +use tonic::{ + codec::{DecodeBuf, Decoder}, + Status, +}; + +#[derive(Clone)] +pub struct MockBody { + data: VecDeque, +} + +impl MockBody { + pub fn new(data: Vec) -> Self { + let mut queue: VecDeque = VecDeque::with_capacity(16); + for msg in data { + let buf = Self::encode(msg); + queue.push_back(buf); + } + + MockBody { data: queue } + } + + pub fn len(&self) -> usize { + self.data.len() + } + + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + + // see: https://github.com/hyperium/tonic/blob/1b03ece2a81cb7e8b1922b3c3c1f496bd402d76c/tonic/src/codec/encode.rs#L52 + fn encode(msg: impl Message) -> Bytes { + let mut buf = BytesMut::with_capacity(256); + + buf.reserve(5); + unsafe { + buf.advance_mut(5); + } + msg.encode(&mut buf).unwrap(); + { + let len = buf.len() - 5; + let mut buf = &mut buf[..5]; + buf.put_u8(0); // byte must be 0, reserve doesn't auto-zero + buf.put_u32(len as u32); + } + buf.freeze() + } +} + +impl Body for MockBody { + type Data = Bytes; + type Error = Status; + + fn poll_data( + mut self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll>> { + if !self.is_empty() { + let msg = self.data.pop_front().unwrap(); + Poll::Ready(Some(Ok(msg))) + } else { + Poll::Ready(None) + } + } + + fn poll_trailers( + self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll, Self::Error>> { + Poll::Ready(Ok(None)) + } +} +/// A [`Decoder`] that knows how to decode `U`. +#[derive(Debug, Clone, Default)] +pub struct ProstDecoder(PhantomData); + +impl ProstDecoder { + pub fn new() -> Self { + Self(PhantomData) + } +} + +impl Decoder for ProstDecoder { + type Item = U; + type Error = Status; + + fn decode(&mut self, buf: &mut DecodeBuf<'_>) -> Result, Self::Error> { + let item = Message::decode(buf.chunk()) + .map(Option::Some) + .map_err(|e| Status::internal(e.to_string()))?; + + buf.advance(buf.chunk().len()); + Ok(item) + } +} diff --git a/osprey_coordinator/src/tonic_mock/mod.rs b/osprey_coordinator/src/tonic_mock/mod.rs new file mode 100644 index 0000000..c89ebe9 --- /dev/null +++ b/osprey_coordinator/src/tonic_mock/mod.rs @@ -0,0 +1,137 @@ +//! This crate is a vendored fork of https://github.com/rnw159/tonic-mock which is not maintained anymore. +//! We keep this fork so we can push our version of tonic forward +use futures::{Stream, StreamExt}; +use prost::Message; +use std::pin::Pin; +use tonic::{Request, Response, Status, Streaming}; + +mod mock; + +pub use mock::{MockBody, ProstDecoder}; + +pub type StreamResponseInner = Pin> + Send + Sync>>; +pub type StreamResponse = Response>; + +/// Generate streaming request for GRPC +/// +/// When testing streaming RPC implemented with tonic, it is pretty clumsy +/// to build the streaming request, this function extracted test code and prost +/// decoder from tonic source code and wrap it with a nice interface. With it, +/// testing your streaming RPC implementation is much easier. +/// +/// Usage: +/// ``` +/// use bytes::Bytes; +/// use prost::Message; +/// use tonic_mock::streaming_request; +/// +/// // normally this should be generated from protos with prost +/// #[derive(Clone, PartialEq, Message)] +/// pub struct Event { +/// #[prost(bytes = "bytes", tag = "1")] +/// pub id: Bytes, +/// #[prost(bytes = "bytes", tag = "2")] +/// pub data: Bytes, +/// } +/// +/// let event = Event { id: Bytes::from("1"), data: Bytes::from("a".repeat(10)) }; +/// let mut events = vec![event.clone(), event.clone(), event]; +/// let stream = tonic_mock::streaming_request(events); +/// +pub fn streaming_request(messages: Vec) -> Request> +where + T: Message + Default + 'static, +{ + let body = MockBody::new(messages); + let decoder: ProstDecoder = ProstDecoder::new(); + let stream = Streaming::new_request(decoder, body, None, None); + + Request::new(stream) +} + +/// a simple wrapper to process and validate streaming response +/// +/// Usage: +/// ``` +/// use tonic::{Response, Status}; +/// use futures::Stream; +/// use std::pin::Pin; +/// +/// #[derive(Clone, PartialEq, ::prost::Message)] +/// pub struct ResponsePush { +/// #[prost(int32, tag = "1")] +/// pub code: i32, +/// } +/// +/// // below code is to mimic a stream response from a GRPC service +/// let output = async_stream::try_stream! { +/// yield ResponsePush { code: 0 }; +/// yield ResponsePush { code: 1 }; +/// yield ResponsePush { code: 2 }; +/// }; +/// let response = Response::new(Box::pin(output) as tonic_mock::StreamResponseInner); +/// let rt = tokio::runtime::Runtime::new().unwrap(); +/// +/// // now we process the events +/// rt.block_on(async { +/// tonic_mock::process_streaming_response(response, |msg, i| { +/// assert!(msg.is_ok()); +/// assert_eq!(msg.as_ref().unwrap().code, i as i32); +/// }).await; +/// }); +/// ``` + +pub async fn process_streaming_response(response: StreamResponse, f: F) +where + T: Message + Default + 'static, + F: Fn(Result, usize), +{ + let mut i: usize = 0; + let mut messages = response.into_inner(); + while let Some(v) = messages.next().await { + f(v, i); + i += 1; + } +} + +/// convert a streaming response to a Vec for simplified testing +/// +/// Usage: +/// ``` +/// use tonic::{Response, Status}; +/// use futures::Stream; +/// use std::pin::Pin; +/// +/// #[derive(Clone, PartialEq, ::prost::Message)] +/// pub struct ResponsePush { +/// #[prost(int32, tag = "1")] +/// pub code: i32, +/// } +/// +/// // below code is to mimic a stream response from a GRPC service +/// let output = async_stream::try_stream! { +/// yield ResponsePush { code: 0 }; +/// yield ResponsePush { code: 1 }; +/// yield ResponsePush { code: 2 }; +/// }; +/// let response = Response::new(Box::pin(output) as tonic_mock::StreamResponseInner); +/// let rt = tokio::runtime::Runtime::new().unwrap(); +/// +/// // now we convert response to vec +/// let result: Vec> = rt.block_on(async { tonic_mock::stream_to_vec(response).await }); +/// for (i, v) in result.iter().enumerate() { +/// assert!(v.is_ok()); +/// assert_eq!(v.as_ref().unwrap().code, i as i32); +/// } +/// ``` +pub async fn stream_to_vec(response: StreamResponse) -> Vec> +where + T: Message + Default + 'static, +{ + let mut result = Vec::new(); + let mut messages = response.into_inner(); + while let Some(v) = messages.next().await { + result.push(v) + } + result +} diff --git a/proto/etcd_watcherd/v1/BUILD b/proto/etcd_watcherd/v1/BUILD new file mode 100644 index 0000000..c14a7f3 --- /dev/null +++ b/proto/etcd_watcherd/v1/BUILD @@ -0,0 +1,12 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "etcd_watcherd_proto", + srcs = ["etcd_watcherd.proto"], + strip_import_prefix = "/proto/", + deps = [ + "//proto/google:com_google_protobuf", + ], +) diff --git a/proto/etcd_watcherd/v1/etcd_watcherd.proto b/proto/etcd_watcherd/v1/etcd_watcherd.proto new file mode 100644 index 0000000..c2eb9ae --- /dev/null +++ b/proto/etcd_watcherd/v1/etcd_watcherd.proto @@ -0,0 +1,49 @@ +syntax = "proto3"; + +package discord_common.etcd_watcherd.v1; + +import "google/protobuf/wrappers.proto"; + +service EtcdWatcherdService +{ + rpc WatchKey(WatchKeyRequest) returns (stream WatchKeyResponse); + rpc WatchKeyRecursive(WatchKeyRecursiveRequest) returns (stream WatchKeyRecursiveResponse); +} + +message WatchKeyRequest +{ + string key = 1; +} + +message WatchKeyResponse +{ + google.protobuf.StringValue value = 1; +} +message WatchKeyRecursiveRequest +{ + string key = 1; +} + +message WatchKeyRecursiveResponse +{ + message FullSync { map items = 1; } + + message SyncOne + { + string key = 1; + string value = 2; + } + + message DeleteOne + { + string key = 1; + string prev_value = 2; + } + + oneof event + { + FullSync full_sync = 1; + SyncOne sync_one = 2; + DeleteOne delete_one = 3; + } +} diff --git a/proto/google/BUILD b/proto/google/BUILD new file mode 100644 index 0000000..b9079dd --- /dev/null +++ b/proto/google/BUILD @@ -0,0 +1,49 @@ +# !!! +# YOU ALMOST CERTAINLY DO NOT WANT TO ADD LIBRARIES TO THIS FILE! +# This directory mirrors the Google official protobuf repos, e.g: +# * https://github.com/protocolbuffers/protobuf/tree/main/src/google +# * https://github.com/googleapis/googleapis/tree/master/google +# The BAZEL file structure used in those repos should be used in this local +# copy (directory) as well, i.e. libraries should be added to BUILD files +# within the directory that contains the `.proto`s themselves +# (and not at the top-level directory). +# !!! + +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "com_google_protobuf", + srcs = glob(["protobuf/**/*.proto"]), + strip_import_prefix = "/proto/", +) + +filegroup( + name = "google_protobuf_files", + srcs = glob(["protobuf/**/*.proto"]), +) + +proto_library( + name = "googleapis", + srcs = glob(["api/**/*.proto"]), + strip_import_prefix = "/proto/", + deps = [ + ":com_google_protobuf", + ], +) + +proto_library( + name = "google_rpc", + srcs = glob(["rpc/**/*.proto"]), + strip_import_prefix = "/proto/", + deps = [ + ":com_google_protobuf", + ], +) + +java_proto_library( + name = "com_google_protobuf_java", + visibility = ["//visibility:public"], + deps = [":com_google_protobuf"], +) diff --git a/proto/google/api/annotations.proto b/proto/google/api/annotations.proto new file mode 100644 index 0000000..efdab3d --- /dev/null +++ b/proto/google/api/annotations.proto @@ -0,0 +1,31 @@ +// Copyright 2015 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/api/http.proto"; +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "AnnotationsProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // See `HttpRule`. + HttpRule http = 72295728; +} diff --git a/proto/google/api/client.proto b/proto/google/api/client.proto new file mode 100644 index 0000000..3b3fd0c --- /dev/null +++ b/proto/google/api/client.proto @@ -0,0 +1,99 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "ClientProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // A definition of a client library method signature. + // + // In client libraries, each proto RPC corresponds to one or more methods + // which the end user is able to call, and calls the underlying RPC. + // Normally, this method receives a single argument (a struct or instance + // corresponding to the RPC request object). Defining this field will + // add one or more overloads providing flattened or simpler method signatures + // in some languages. + // + // The fields on the method signature are provided as a comma-separated + // string. + // + // For example, the proto RPC and annotation: + // + // rpc CreateSubscription(CreateSubscriptionRequest) + // returns (Subscription) { + // option (google.api.method_signature) = "name,topic"; + // } + // + // Would add the following Java overload (in addition to the method accepting + // the request object): + // + // public final Subscription createSubscription(String name, String topic) + // + // The following backwards-compatibility guidelines apply: + // + // * Adding this annotation to an unannotated method is backwards + // compatible. + // * Adding this annotation to a method which already has existing + // method signature annotations is backwards compatible if and only if + // the new method signature annotation is last in the sequence. + // * Modifying or removing an existing method signature annotation is + // a breaking change. + // * Re-ordering existing method signature annotations is a breaking + // change. + repeated string method_signature = 1051; +} + +extend google.protobuf.ServiceOptions { + // The hostname for this service. + // This should be specified with no prefix or protocol. + // + // Example: + // + // service Foo { + // option (google.api.default_host) = "foo.googleapi.com"; + // ... + // } + string default_host = 1049; + + // OAuth scopes needed for the client. + // + // Example: + // + // service Foo { + // option (google.api.oauth_scopes) = \ + // "https://www.googleapis.com/auth/cloud-platform"; + // ... + // } + // + // If there is more than one scope, use a comma-separated string: + // + // Example: + // + // service Foo { + // option (google.api.oauth_scopes) = \ + // "https://www.googleapis.com/auth/cloud-platform," + // "https://www.googleapis.com/auth/monitoring"; + // ... + // } + string oauth_scopes = 1050; +} diff --git a/proto/google/api/field_behavior.proto b/proto/google/api/field_behavior.proto new file mode 100644 index 0000000..c4abe3b --- /dev/null +++ b/proto/google/api/field_behavior.proto @@ -0,0 +1,90 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "FieldBehaviorProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.FieldOptions { + // A designation of a specific field behavior (required, output only, etc.) + // in protobuf messages. + // + // Examples: + // + // string name = 1 [(google.api.field_behavior) = REQUIRED]; + // State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + // google.protobuf.Duration ttl = 1 + // [(google.api.field_behavior) = INPUT_ONLY]; + // google.protobuf.Timestamp expire_time = 1 + // [(google.api.field_behavior) = OUTPUT_ONLY, + // (google.api.field_behavior) = IMMUTABLE]; + repeated google.api.FieldBehavior field_behavior = 1052; +} + +// An indicator of the behavior of a given field (for example, that a field +// is required in requests, or given as output but ignored as input). +// This **does not** change the behavior in protocol buffers itself; it only +// denotes the behavior and may affect how API tooling handles the field. +// +// Note: This enum **may** receive new values in the future. +enum FieldBehavior { + // Conventional default for enums. Do not use this. + FIELD_BEHAVIOR_UNSPECIFIED = 0; + + // Specifically denotes a field as optional. + // While all fields in protocol buffers are optional, this may be specified + // for emphasis if appropriate. + OPTIONAL = 1; + + // Denotes a field as required. + // This indicates that the field **must** be provided as part of the request, + // and failure to do so will cause an error (usually `INVALID_ARGUMENT`). + REQUIRED = 2; + + // Denotes a field as output only. + // This indicates that the field is provided in responses, but including the + // field in a request does nothing (the server *must* ignore it and + // *must not* throw an error as a result of the field's presence). + OUTPUT_ONLY = 3; + + // Denotes a field as input only. + // This indicates that the field is provided in requests, and the + // corresponding field is not included in output. + INPUT_ONLY = 4; + + // Denotes a field as immutable. + // This indicates that the field may be set once in a request to create a + // resource, but may not be changed thereafter. + IMMUTABLE = 5; + + // Denotes that a (repeated) field is an unordered list. + // This indicates that the service may provide the elements of the list + // in any arbitrary order, rather than the order the user originally + // provided. Additionally, the list's order may or may not be stable. + UNORDERED_LIST = 6; + + // Denotes that this field returns a non-empty default value if not set. + // This indicates that if the user provides the empty value in a request, + // a non-empty value will be returned. The user will not be aware of what + // non-empty value to expect. + NON_EMPTY_DEFAULT = 7; +} diff --git a/proto/google/api/http.proto b/proto/google/api/http.proto new file mode 100644 index 0000000..b2977f5 --- /dev/null +++ b/proto/google/api/http.proto @@ -0,0 +1,376 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.api; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "HttpProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +// Defines the HTTP configuration for an API service. It contains a list of +// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method +// to one or more HTTP REST API methods. +message Http { + // A list of HTTP configuration rules that apply to individual API methods. + // + // **NOTE:** All service configuration rules follow "last one wins" order. + repeated HttpRule rules = 1; + + // When set to true, URL path parameters will be fully URI-decoded except in + // cases of single segment matches in reserved expansion, where "%2F" will be + // left encoded. + // + // The default behavior is to not decode RFC 6570 reserved characters in multi + // segment matches. + bool fully_decode_reserved_expansion = 2; +} + +// # gRPC Transcoding +// +// gRPC Transcoding is a feature for mapping between a gRPC method and one or +// more HTTP REST endpoints. It allows developers to build a single API service +// that supports both gRPC APIs and REST APIs. Many systems, including [Google +// APIs](https://github.com/googleapis/googleapis), +// [Cloud Endpoints](https://cloud.google.com/endpoints), [gRPC +// Gateway](https://github.com/grpc-ecosystem/grpc-gateway), +// and [Envoy](https://github.com/envoyproxy/envoy) proxy support this feature +// and use it for large scale production services. +// +// `HttpRule` defines the schema of the gRPC/REST mapping. The mapping specifies +// how different portions of the gRPC request message are mapped to the URL +// path, URL query parameters, and HTTP request body. It also controls how the +// gRPC response message is mapped to the HTTP response body. `HttpRule` is +// typically specified as an `google.api.http` annotation on the gRPC method. +// +// Each mapping specifies a URL path template and an HTTP method. The path +// template may refer to one or more fields in the gRPC request message, as long +// as each field is a non-repeated field with a primitive (non-message) type. +// The path template controls how fields of the request message are mapped to +// the URL path. +// +// Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/{name=messages/*}" +// }; +// } +// } +// message GetMessageRequest { +// string name = 1; // Mapped to URL path. +// } +// message Message { +// string text = 1; // The resource content. +// } +// +// This enables an HTTP REST to gRPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(name: "messages/123456")` +// +// Any fields in the request message which are not bound by the path template +// automatically become HTTP query parameters if there is no HTTP request body. +// For example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get:"/v1/messages/{message_id}" +// }; +// } +// } +// message GetMessageRequest { +// message SubMessage { +// string subfield = 1; +// } +// string message_id = 1; // Mapped to URL path. +// int64 revision = 2; // Mapped to URL query parameter `revision`. +// SubMessage sub = 3; // Mapped to URL query parameter `sub.subfield`. +// } +// +// This enables a HTTP JSON to RPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456?revision=2&sub.subfield=foo` | +// `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: +// "foo"))` +// +// Note that fields which are mapped to URL query parameters must have a +// primitive type or a repeated primitive type or a non-repeated message type. +// In the case of a repeated type, the parameter can be repeated in the URL +// as `...?param=A¶m=B`. In the case of a message type, each field of the +// message is mapped to a separate parameter, such as +// `...?foo.a=A&foo.b=B&foo.c=C`. +// +// For HTTP methods that allow a request body, the `body` field +// specifies the mapping. Consider a REST update method on the +// message resource collection: +// +// service Messaging { +// rpc UpdateMessage(UpdateMessageRequest) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "message" +// }; +// } +// } +// message UpdateMessageRequest { +// string message_id = 1; // mapped to the URL +// Message message = 2; // mapped to the body +// } +// +// The following HTTP JSON to RPC mapping is enabled, where the +// representation of the JSON in the request body is determined by +// protos JSON encoding: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" message { text: "Hi!" })` +// +// The special name `*` can be used in the body mapping to define that +// every field not bound by the path template should be mapped to the +// request body. This enables the following alternative definition of +// the update method: +// +// service Messaging { +// rpc UpdateMessage(Message) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "*" +// }; +// } +// } +// message Message { +// string message_id = 1; +// string text = 2; +// } +// +// +// The following HTTP JSON to RPC mapping is enabled: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" text: "Hi!")` +// +// Note that when using `*` in the body mapping, it is not possible to +// have HTTP parameters, as all fields not bound by the path end in +// the body. This makes this option more rarely used in practice when +// defining REST APIs. The common usage of `*` is in custom methods +// which don't use the URL at all for transferring data. +// +// It is possible to define multiple HTTP methods for one RPC by using +// the `additional_bindings` option. Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/messages/{message_id}" +// additional_bindings { +// get: "/v1/users/{user_id}/messages/{message_id}" +// } +// }; +// } +// } +// message GetMessageRequest { +// string message_id = 1; +// string user_id = 2; +// } +// +// This enables the following two alternative HTTP JSON to RPC mappings: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")` +// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: +// "123456")` +// +// ## Rules for HTTP mapping +// +// 1. Leaf request fields (recursive expansion nested messages in the request +// message) are classified into three categories: +// - Fields referred by the path template. They are passed via the URL path. +// - Fields referred by the [HttpRule.body][google.api.HttpRule.body]. They are passed via the HTTP +// request body. +// - All other fields are passed via the URL query parameters, and the +// parameter name is the field path in the request message. A repeated +// field can be represented as multiple query parameters under the same +// name. +// 2. If [HttpRule.body][google.api.HttpRule.body] is "*", there is no URL query parameter, all fields +// are passed via URL path and HTTP request body. +// 3. If [HttpRule.body][google.api.HttpRule.body] is omitted, there is no HTTP request body, all +// fields are passed via URL path and URL query parameters. +// +// ### Path template syntax +// +// Template = "/" Segments [ Verb ] ; +// Segments = Segment { "/" Segment } ; +// Segment = "*" | "**" | LITERAL | Variable ; +// Variable = "{" FieldPath [ "=" Segments ] "}" ; +// FieldPath = IDENT { "." IDENT } ; +// Verb = ":" LITERAL ; +// +// The syntax `*` matches a single URL path segment. The syntax `**` matches +// zero or more URL path segments, which must be the last part of the URL path +// except the `Verb`. +// +// The syntax `Variable` matches part of the URL path as specified by its +// template. A variable template must not contain other variables. If a variable +// matches a single path segment, its template may be omitted, e.g. `{var}` +// is equivalent to `{var=*}`. +// +// The syntax `LITERAL` matches literal text in the URL path. If the `LITERAL` +// contains any reserved character, such characters should be percent-encoded +// before the matching. +// +// If a variable contains exactly one path segment, such as `"{var}"` or +// `"{var=*}"`, when such a variable is expanded into a URL path on the client +// side, all characters except `[-_.~0-9a-zA-Z]` are percent-encoded. The +// server side does the reverse decoding. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{var}`. +// +// If a variable contains multiple path segments, such as `"{var=foo/*}"` +// or `"{var=**}"`, when such a variable is expanded into a URL path on the +// client side, all characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. +// The server side does the reverse decoding, except "%2F" and "%2f" are left +// unchanged. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{+var}`. +// +// ## Using gRPC API Service Configuration +// +// gRPC API Service Configuration (service config) is a configuration language +// for configuring a gRPC service to become a user-facing product. The +// service config is simply the YAML representation of the `google.api.Service` +// proto message. +// +// As an alternative to annotating your proto file, you can configure gRPC +// transcoding in your service config YAML files. You do this by specifying a +// `HttpRule` that maps the gRPC method to a REST endpoint, achieving the same +// effect as the proto annotation. This can be particularly useful if you +// have a proto that is reused in multiple services. Note that any transcoding +// specified in the service config will override any matching transcoding +// configuration in the proto. +// +// Example: +// +// http: +// rules: +// # Selects a gRPC method and applies HttpRule to it. +// - selector: example.v1.Messaging.GetMessage +// get: /v1/messages/{message_id}/{sub.subfield} +// +// ## Special notes +// +// When gRPC Transcoding is used to map a gRPC to JSON REST endpoints, the +// proto to JSON conversion must follow the [proto3 +// specification](https://developers.google.com/protocol-buffers/docs/proto3#json). +// +// While the single segment variable follows the semantics of +// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String +// Expansion, the multi segment variable **does not** follow RFC 6570 Section +// 3.2.3 Reserved Expansion. The reason is that the Reserved Expansion +// does not expand special characters like `?` and `#`, which would lead +// to invalid URLs. As the result, gRPC Transcoding uses a custom encoding +// for multi segment variables. +// +// The path variables **must not** refer to any repeated or mapped field, +// because client libraries are not capable of handling such variable expansion. +// +// The path variables **must not** capture the leading "/" character. The reason +// is that the most common use case "{var}" does not capture the leading "/" +// character. For consistency, all path variables must share the same behavior. +// +// Repeated message fields must not be mapped to URL query parameters, because +// no client library can support such complicated mapping. +// +// If an API needs to use a JSON array for request or response body, it can map +// the request or response body to a repeated field. However, some gRPC +// Transcoding implementations may not support this feature. +message HttpRule { + // Selects a method to which this rule applies. + // + // Refer to [selector][google.api.DocumentationRule.selector] for syntax details. + string selector = 1; + + // Determines the URL pattern is matched by this rules. This pattern can be + // used with any of the {get|put|post|delete|patch} methods. A custom method + // can be defined using the 'custom' field. + oneof pattern { + // Maps to HTTP GET. Used for listing and getting information about + // resources. + string get = 2; + + // Maps to HTTP PUT. Used for replacing a resource. + string put = 3; + + // Maps to HTTP POST. Used for creating a resource or performing an action. + string post = 4; + + // Maps to HTTP DELETE. Used for deleting a resource. + string delete = 5; + + // Maps to HTTP PATCH. Used for updating a resource. + string patch = 6; + + // The custom pattern is used for specifying an HTTP method that is not + // included in the `pattern` field, such as HEAD, or "*" to leave the + // HTTP method unspecified for this rule. The wild-card rule is useful + // for services that provide content to Web (HTML) clients. + CustomHttpPattern custom = 8; + } + + // The name of the request field whose value is mapped to the HTTP request + // body, or `*` for mapping all request fields not captured by the path + // pattern to the HTTP body, or omitted for not having any HTTP request body. + // + // NOTE: the referred field must be present at the top-level of the request + // message type. + string body = 7; + + // Optional. The name of the response field whose value is mapped to the HTTP + // response body. When omitted, the entire response message will be used + // as the HTTP response body. + // + // NOTE: The referred field must be present at the top-level of the response + // message type. + string response_body = 12; + + // Additional HTTP bindings for the selector. Nested bindings must + // not contain an `additional_bindings` field themselves (that is, + // the nesting may only be one level deep). + repeated HttpRule additional_bindings = 11; +} + +// A custom pattern is used for defining custom HTTP verb. +message CustomHttpPattern { + // The name of this custom HTTP verb. + string kind = 1; + + // The path matched by this custom verb. + string path = 2; +} diff --git a/proto/google/api/resource.proto b/proto/google/api/resource.proto new file mode 100644 index 0000000..2dfff15 --- /dev/null +++ b/proto/google/api/resource.proto @@ -0,0 +1,299 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/protobuf/descriptor.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "ResourceProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.FieldOptions { + // An annotation that describes a resource reference, see + // [ResourceReference][]. + google.api.ResourceReference resource_reference = 1055; +} + +extend google.protobuf.FileOptions { + // An annotation that describes a resource definition without a corresponding + // message; see [ResourceDescriptor][]. + repeated google.api.ResourceDescriptor resource_definition = 1053; +} + +extend google.protobuf.MessageOptions { + // An annotation that describes a resource definition, see + // [ResourceDescriptor][]. + google.api.ResourceDescriptor resource = 1053; +} + +// A simple descriptor of a resource type. +// +// ResourceDescriptor annotates a resource message (either by means of a +// protobuf annotation or use in the service config), and associates the +// resource's schema, the resource type, and the pattern of the resource name. +// +// Example: +// +// message Topic { +// // Indicates this message defines a resource schema. +// // Declares the resource type in the format of {service}/{kind}. +// // For Kubernetes resources, the format is {api group}/{kind}. +// option (google.api.resource) = { +// type: "pubsub.googleapis.com/Topic" +// name_descriptor: { +// pattern: "projects/{project}/topics/{topic}" +// parent_type: "cloudresourcemanager.googleapis.com/Project" +// parent_name_extractor: "projects/{project}" +// } +// }; +// } +// +// The ResourceDescriptor Yaml config will look like: +// +// resources: +// - type: "pubsub.googleapis.com/Topic" +// name_descriptor: +// - pattern: "projects/{project}/topics/{topic}" +// parent_type: "cloudresourcemanager.googleapis.com/Project" +// parent_name_extractor: "projects/{project}" +// +// Sometimes, resources have multiple patterns, typically because they can +// live under multiple parents. +// +// Example: +// +// message LogEntry { +// option (google.api.resource) = { +// type: "logging.googleapis.com/LogEntry" +// name_descriptor: { +// pattern: "projects/{project}/logs/{log}" +// parent_type: "cloudresourcemanager.googleapis.com/Project" +// parent_name_extractor: "projects/{project}" +// } +// name_descriptor: { +// pattern: "folders/{folder}/logs/{log}" +// parent_type: "cloudresourcemanager.googleapis.com/Folder" +// parent_name_extractor: "folders/{folder}" +// } +// name_descriptor: { +// pattern: "organizations/{organization}/logs/{log}" +// parent_type: "cloudresourcemanager.googleapis.com/Organization" +// parent_name_extractor: "organizations/{organization}" +// } +// name_descriptor: { +// pattern: "billingAccounts/{billing_account}/logs/{log}" +// parent_type: "billing.googleapis.com/BillingAccount" +// parent_name_extractor: "billingAccounts/{billing_account}" +// } +// }; +// } +// +// The ResourceDescriptor Yaml config will look like: +// +// resources: +// - type: 'logging.googleapis.com/LogEntry' +// name_descriptor: +// - pattern: "projects/{project}/logs/{log}" +// parent_type: "cloudresourcemanager.googleapis.com/Project" +// parent_name_extractor: "projects/{project}" +// - pattern: "folders/{folder}/logs/{log}" +// parent_type: "cloudresourcemanager.googleapis.com/Folder" +// parent_name_extractor: "folders/{folder}" +// - pattern: "organizations/{organization}/logs/{log}" +// parent_type: "cloudresourcemanager.googleapis.com/Organization" +// parent_name_extractor: "organizations/{organization}" +// - pattern: "billingAccounts/{billing_account}/logs/{log}" +// parent_type: "billing.googleapis.com/BillingAccount" +// parent_name_extractor: "billingAccounts/{billing_account}" +// +// For flexible resources, the resource name doesn't contain parent names, but +// the resource itself has parents for policy evaluation. +// +// Example: +// +// message Shelf { +// option (google.api.resource) = { +// type: "library.googleapis.com/Shelf" +// name_descriptor: { +// pattern: "shelves/{shelf}" +// parent_type: "cloudresourcemanager.googleapis.com/Project" +// } +// name_descriptor: { +// pattern: "shelves/{shelf}" +// parent_type: "cloudresourcemanager.googleapis.com/Folder" +// } +// }; +// } +// +// The ResourceDescriptor Yaml config will look like: +// +// resources: +// - type: 'library.googleapis.com/Shelf' +// name_descriptor: +// - pattern: "shelves/{shelf}" +// parent_type: "cloudresourcemanager.googleapis.com/Project" +// - pattern: "shelves/{shelf}" +// parent_type: "cloudresourcemanager.googleapis.com/Folder" +message ResourceDescriptor { + // A description of the historical or future-looking state of the + // resource pattern. + enum History { + // The "unset" value. + HISTORY_UNSPECIFIED = 0; + + // The resource originally had one pattern and launched as such, and + // additional patterns were added later. + ORIGINALLY_SINGLE_PATTERN = 1; + + // The resource has one pattern, but the API owner expects to add more + // later. (This is the inverse of ORIGINALLY_SINGLE_PATTERN, and prevents + // that from being necessary once there are multiple patterns.) + FUTURE_MULTI_PATTERN = 2; + } + + // A flag representing a specific style that a resource claims to conform to. + enum Style { + // The unspecified value. Do not use. + STYLE_UNSPECIFIED = 0; + + // This resource is intended to be "declarative-friendly". + // + // Declarative-friendly resources must be more strictly consistent, and + // setting this to true communicates to tools that this resource should + // adhere to declarative-friendly expectations. + // + // Note: This is used by the API linter (linter.aip.dev) to enable + // additional checks. + DECLARATIVE_FRIENDLY = 1; + } + + // The resource type. It must be in the format of + // {service_name}/{resource_type_kind}. The `resource_type_kind` must be + // singular and must not include version numbers. + // + // Example: `storage.googleapis.com/Bucket` + // + // The value of the resource_type_kind must follow the regular expression + // /[A-Za-z][a-zA-Z0-9]+/. It should start with an upper case character and + // should use PascalCase (UpperCamelCase). The maximum number of + // characters allowed for the `resource_type_kind` is 100. + string type = 1; + + // Optional. The relative resource name pattern associated with this resource + // type. The DNS prefix of the full resource name shouldn't be specified here. + // + // The path pattern must follow the syntax, which aligns with HTTP binding + // syntax: + // + // Template = Segment { "/" Segment } ; + // Segment = LITERAL | Variable ; + // Variable = "{" LITERAL "}" ; + // + // Examples: + // + // - "projects/{project}/topics/{topic}" + // - "projects/{project}/knowledgeBases/{knowledge_base}" + // + // The components in braces correspond to the IDs for each resource in the + // hierarchy. It is expected that, if multiple patterns are provided, + // the same component name (e.g. "project") refers to IDs of the same + // type of resource. + repeated string pattern = 2; + + // Optional. The field on the resource that designates the resource name + // field. If omitted, this is assumed to be "name". + string name_field = 3; + + // Optional. The historical or future-looking state of the resource pattern. + // + // Example: + // + // // The InspectTemplate message originally only supported resource + // // names with organization, and project was added later. + // message InspectTemplate { + // option (google.api.resource) = { + // type: "dlp.googleapis.com/InspectTemplate" + // pattern: + // "organizations/{organization}/inspectTemplates/{inspect_template}" + // pattern: "projects/{project}/inspectTemplates/{inspect_template}" + // history: ORIGINALLY_SINGLE_PATTERN + // }; + // } + History history = 4; + + // The plural name used in the resource name and permission names, such as + // 'projects' for the resource name of 'projects/{project}' and the permission + // name of 'cloudresourcemanager.googleapis.com/projects.get'. It is the same + // concept of the `plural` field in k8s CRD spec + // https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + // + // Note: The plural form is required even for singleton resources. See + // https://aip.dev/156 + string plural = 5; + + // The same concept of the `singular` field in k8s CRD spec + // https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + // Such as "project" for the `resourcemanager.googleapis.com/Project` type. + string singular = 6; + + // Style flag(s) for this resource. + // These indicate that a resource is expected to conform to a given + // style. See the specific style flags for additional information. + repeated Style style = 10; +} + +// Defines a proto annotation that describes a string field that refers to +// an API resource. +message ResourceReference { + // The resource type that the annotated field references. + // + // Example: + // + // message Subscription { + // string topic = 2 [(google.api.resource_reference) = { + // type: "pubsub.googleapis.com/Topic" + // }]; + // } + // + // Occasionally, a field may reference an arbitrary resource. In this case, + // APIs use the special value * in their resource reference. + // + // Example: + // + // message GetIamPolicyRequest { + // string resource = 2 [(google.api.resource_reference) = { + // type: "*" + // }]; + // } + string type = 1; + + // The resource type of a child collection that the annotated field + // references. This is useful for annotating the `parent` field that + // doesn't have a fixed resource type. + // + // Example: + // + // message ListLogEntriesRequest { + // string parent = 1 [(google.api.resource_reference) = { + // child_type: "logging.googleapis.com/LogEntry" + // }; + // } + string child_type = 2; +} diff --git a/proto/google/bigtable/v2/BUILD b/proto/google/bigtable/v2/BUILD new file mode 100644 index 0000000..5e13ccf --- /dev/null +++ b/proto/google/bigtable/v2/BUILD @@ -0,0 +1,17 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "bigtable_proto", + srcs = [ + "bigtable.proto", + "data.proto", + ], + strip_import_prefix = "/proto/", + deps = [ + "//proto/google:com_google_protobuf", + "//proto/google:google_rpc", + "//proto/google:googleapis", + ], +) diff --git a/proto/google/bigtable/v2/bigtable.proto b/proto/google/bigtable/v2/bigtable.proto new file mode 100644 index 0000000..32aaba2 --- /dev/null +++ b/proto/google/bigtable/v2/bigtable.proto @@ -0,0 +1,427 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.v2; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/bigtable/v2/data.proto"; +import "google/protobuf/wrappers.proto"; +import "google/rpc/status.proto"; + +option csharp_namespace = "Google.Cloud.Bigtable.V2"; +option go_package = "google.golang.org/genproto/googleapis/bigtable/v2;bigtable"; +option java_multiple_files = true; +option java_outer_classname = "BigtableProto"; +option java_package = "com.google.bigtable.v2"; +option php_namespace = "Google\\Cloud\\Bigtable\\V2"; +option ruby_package = "Google::Cloud::Bigtable::V2"; +option (google.api.resource_definition) = { + type: "bigtable.googleapis.com/Table" + pattern: "projects/{project}/instances/{instance}/tables/{table}" +}; + +// Service for reading from and writing to existing Bigtable tables. +service Bigtable { + option (google.api.default_host) = "bigtable.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/bigtable.data," + "https://www.googleapis.com/auth/bigtable.data.readonly," + "https://www.googleapis.com/auth/cloud-bigtable.data," + "https://www.googleapis.com/auth/cloud-bigtable.data.readonly," + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/cloud-platform.read-only"; + + // Streams back the contents of all requested rows in key order, optionally + // applying the same Reader filter to each. Depending on their size, + // rows and cells may be broken up across multiple responses, but + // atomicity of each row will still be preserved. See the + // ReadRowsResponse documentation for details. + rpc ReadRows(ReadRowsRequest) returns (stream ReadRowsResponse) { + option (google.api.http) = { + post: "/v2/{table_name=projects/*/instances/*/tables/*}:readRows" + body: "*" + }; + option (google.api.method_signature) = "table_name"; + option (google.api.method_signature) = "table_name,app_profile_id"; + } + + // Returns a sample of row keys in the table. The returned row keys will + // delimit contiguous sections of the table of approximately equal size, + // which can be used to break up the data for distributed tasks like + // mapreduces. + rpc SampleRowKeys(SampleRowKeysRequest) returns (stream SampleRowKeysResponse) { + option (google.api.http) = { + get: "/v2/{table_name=projects/*/instances/*/tables/*}:sampleRowKeys" + }; + option (google.api.method_signature) = "table_name"; + option (google.api.method_signature) = "table_name,app_profile_id"; + } + + // Mutates a row atomically. Cells already present in the row are left + // unchanged unless explicitly changed by `mutation`. + rpc MutateRow(MutateRowRequest) returns (MutateRowResponse) { + option (google.api.http) = { + post: "/v2/{table_name=projects/*/instances/*/tables/*}:mutateRow" + body: "*" + }; + option (google.api.method_signature) = "table_name,row_key,mutations"; + option (google.api.method_signature) = "table_name,row_key,mutations,app_profile_id"; + } + + // Mutates multiple rows in a batch. Each individual row is mutated + // atomically as in MutateRow, but the entire batch is not executed + // atomically. + rpc MutateRows(MutateRowsRequest) returns (stream MutateRowsResponse) { + option (google.api.http) = { + post: "/v2/{table_name=projects/*/instances/*/tables/*}:mutateRows" + body: "*" + }; + option (google.api.method_signature) = "table_name,entries"; + option (google.api.method_signature) = "table_name,entries,app_profile_id"; + } + + // Mutates a row atomically based on the output of a predicate Reader filter. + rpc CheckAndMutateRow(CheckAndMutateRowRequest) returns (CheckAndMutateRowResponse) { + option (google.api.http) = { + post: "/v2/{table_name=projects/*/instances/*/tables/*}:checkAndMutateRow" + body: "*" + }; + option (google.api.method_signature) = "table_name,row_key,predicate_filter,true_mutations,false_mutations"; + option (google.api.method_signature) = "table_name,row_key,predicate_filter,true_mutations,false_mutations,app_profile_id"; + } + + // Modifies a row atomically on the server. The method reads the latest + // existing timestamp and value from the specified columns and writes a new + // entry based on pre-defined read/modify/write rules. The new value for the + // timestamp is the greater of the existing timestamp or the current server + // time. The method returns the new contents of all modified cells. + rpc ReadModifyWriteRow(ReadModifyWriteRowRequest) returns (ReadModifyWriteRowResponse) { + option (google.api.http) = { + post: "/v2/{table_name=projects/*/instances/*/tables/*}:readModifyWriteRow" + body: "*" + }; + option (google.api.method_signature) = "table_name,row_key,rules"; + option (google.api.method_signature) = "table_name,row_key,rules,app_profile_id"; + } +} + +// Request message for Bigtable.ReadRows. +message ReadRowsRequest { + // Required. The unique name of the table from which to read. + // Values are of the form + // `projects//instances//tables/`. + string table_name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigtable.googleapis.com/Table" + } + ]; + + // This value specifies routing for replication. If not specified, the + // "default" application profile will be used. + string app_profile_id = 5; + + // The row keys and/or ranges to read. If not specified, reads from all rows. + RowSet rows = 2; + + // The filter to apply to the contents of the specified row(s). If unset, + // reads the entirety of each row. + RowFilter filter = 3; + + // The read will terminate after committing to N rows' worth of results. The + // default (zero) is to return all results. + int64 rows_limit = 4; +} + +// Response message for Bigtable.ReadRows. +message ReadRowsResponse { + // Specifies a piece of a row's contents returned as part of the read + // response stream. + message CellChunk { + // The row key for this chunk of data. If the row key is empty, + // this CellChunk is a continuation of the same row as the previous + // CellChunk in the response stream, even if that CellChunk was in a + // previous ReadRowsResponse message. + bytes row_key = 1; + + // The column family name for this chunk of data. If this message + // is not present this CellChunk is a continuation of the same column + // family as the previous CellChunk. The empty string can occur as a + // column family name in a response so clients must check + // explicitly for the presence of this message, not just for + // `family_name.value` being non-empty. + google.protobuf.StringValue family_name = 2; + + // The column qualifier for this chunk of data. If this message + // is not present, this CellChunk is a continuation of the same column + // as the previous CellChunk. Column qualifiers may be empty so + // clients must check for the presence of this message, not just + // for `qualifier.value` being non-empty. + google.protobuf.BytesValue qualifier = 3; + + // The cell's stored timestamp, which also uniquely identifies it + // within its column. Values are always expressed in + // microseconds, but individual tables may set a coarser + // granularity to further restrict the allowed values. For + // example, a table which specifies millisecond granularity will + // only allow values of `timestamp_micros` which are multiples of + // 1000. Timestamps are only set in the first CellChunk per cell + // (for cells split into multiple chunks). + int64 timestamp_micros = 4; + + // Labels applied to the cell by a + // [RowFilter][google.bigtable.v2.RowFilter]. Labels are only set + // on the first CellChunk per cell. + repeated string labels = 5; + + // The value stored in the cell. Cell values can be split across + // multiple CellChunks. In that case only the value field will be + // set in CellChunks after the first: the timestamp and labels + // will only be present in the first CellChunk, even if the first + // CellChunk came in a previous ReadRowsResponse. + bytes value = 6; + + // If this CellChunk is part of a chunked cell value and this is + // not the final chunk of that cell, value_size will be set to the + // total length of the cell value. The client can use this size + // to pre-allocate memory to hold the full cell value. + int32 value_size = 7; + + // Signals to the client concerning previous CellChunks received. + oneof row_status { + // Indicates that the client should drop all previous chunks for + // `row_key`, as it will be re-read from the beginning. + bool reset_row = 8; + + // Indicates that the client can safely process all previous chunks for + // `row_key`, as its data has been fully read. + bool commit_row = 9; + } + } + + // A collection of a row's contents as part of the read request. + repeated CellChunk chunks = 1; + + // Optionally the server might return the row key of the last row it + // has scanned. The client can use this to construct a more + // efficient retry request if needed: any row keys or portions of + // ranges less than this row key can be dropped from the request. + // This is primarily useful for cases where the server has read a + // lot of data that was filtered out since the last committed row + // key, allowing the client to skip that work on a retry. + bytes last_scanned_row_key = 2; +} + +// Request message for Bigtable.SampleRowKeys. +message SampleRowKeysRequest { + // Required. The unique name of the table from which to sample row keys. + // Values are of the form + // `projects//instances//tables/
`. + string table_name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigtable.googleapis.com/Table" + } + ]; + + // This value specifies routing for replication. If not specified, the + // "default" application profile will be used. + string app_profile_id = 2; +} + +// Response message for Bigtable.SampleRowKeys. +message SampleRowKeysResponse { + // Sorted streamed sequence of sample row keys in the table. The table might + // have contents before the first row key in the list and after the last one, + // but a key containing the empty string indicates "end of table" and will be + // the last response given, if present. + // Note that row keys in this list may not have ever been written to or read + // from, and users should therefore not make any assumptions about the row key + // structure that are specific to their use case. + bytes row_key = 1; + + // Approximate total storage space used by all rows in the table which precede + // `row_key`. Buffering the contents of all rows between two subsequent + // samples would require space roughly equal to the difference in their + // `offset_bytes` fields. + int64 offset_bytes = 2; +} + +// Request message for Bigtable.MutateRow. +message MutateRowRequest { + // Required. The unique name of the table to which the mutation should be applied. + // Values are of the form + // `projects//instances//tables/
`. + string table_name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigtable.googleapis.com/Table" + } + ]; + + // This value specifies routing for replication. If not specified, the + // "default" application profile will be used. + string app_profile_id = 4; + + // Required. The key of the row to which the mutation should be applied. + bytes row_key = 2 [(google.api.field_behavior) = REQUIRED]; + + // Required. Changes to be atomically applied to the specified row. Entries are applied + // in order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry and at most 100000. + repeated Mutation mutations = 3 [(google.api.field_behavior) = REQUIRED]; +} + +// Response message for Bigtable.MutateRow. +message MutateRowResponse { + +} + +// Request message for BigtableService.MutateRows. +message MutateRowsRequest { + // A mutation for a given row. + message Entry { + // The key of the row to which the `mutations` should be applied. + bytes row_key = 1; + + // Required. Changes to be atomically applied to the specified row. Mutations are + // applied in order, meaning that earlier mutations can be masked by + // later ones. + // You must specify at least one mutation. + repeated Mutation mutations = 2 [(google.api.field_behavior) = REQUIRED]; + } + + // Required. The unique name of the table to which the mutations should be applied. + string table_name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigtable.googleapis.com/Table" + } + ]; + + // This value specifies routing for replication. If not specified, the + // "default" application profile will be used. + string app_profile_id = 3; + + // Required. The row keys and corresponding mutations to be applied in bulk. + // Each entry is applied as an atomic mutation, but the entries may be + // applied in arbitrary order (even between entries for the same row). + // At least one entry must be specified, and in total the entries can + // contain at most 100000 mutations. + repeated Entry entries = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// Response message for BigtableService.MutateRows. +message MutateRowsResponse { + // The result of applying a passed mutation in the original request. + message Entry { + // The index into the original request's `entries` list of the Entry + // for which a result is being reported. + int64 index = 1; + + // The result of the request Entry identified by `index`. + // Depending on how requests are batched during execution, it is possible + // for one Entry to fail due to an error with another Entry. In the event + // that this occurs, the same error will be reported for both entries. + google.rpc.Status status = 2; + } + + // One or more results for Entries from the batch request. + repeated Entry entries = 1; +} + +// Request message for Bigtable.CheckAndMutateRow. +message CheckAndMutateRowRequest { + // Required. The unique name of the table to which the conditional mutation should be + // applied. + // Values are of the form + // `projects//instances//tables/
`. + string table_name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigtable.googleapis.com/Table" + } + ]; + + // This value specifies routing for replication. If not specified, the + // "default" application profile will be used. + string app_profile_id = 7; + + // Required. The key of the row to which the conditional mutation should be applied. + bytes row_key = 2 [(google.api.field_behavior) = REQUIRED]; + + // The filter to be applied to the contents of the specified row. Depending + // on whether or not any results are yielded, either `true_mutations` or + // `false_mutations` will be executed. If unset, checks that the row contains + // any values at all. + RowFilter predicate_filter = 6; + + // Changes to be atomically applied to the specified row if `predicate_filter` + // yields at least one cell when applied to `row_key`. Entries are applied in + // order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry if `false_mutations` is empty, and at most + // 100000. + repeated Mutation true_mutations = 4; + + // Changes to be atomically applied to the specified row if `predicate_filter` + // does not yield any cells when applied to `row_key`. Entries are applied in + // order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry if `true_mutations` is empty, and at most + // 100000. + repeated Mutation false_mutations = 5; +} + +// Response message for Bigtable.CheckAndMutateRow. +message CheckAndMutateRowResponse { + // Whether or not the request's `predicate_filter` yielded any results for + // the specified row. + bool predicate_matched = 1; +} + +// Request message for Bigtable.ReadModifyWriteRow. +message ReadModifyWriteRowRequest { + // Required. The unique name of the table to which the read/modify/write rules should be + // applied. + // Values are of the form + // `projects//instances//tables/
`. + string table_name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigtable.googleapis.com/Table" + } + ]; + + // This value specifies routing for replication. If not specified, the + // "default" application profile will be used. + string app_profile_id = 4; + + // Required. The key of the row to which the read/modify/write rules should be applied. + bytes row_key = 2 [(google.api.field_behavior) = REQUIRED]; + + // Required. Rules specifying how the specified row's contents are to be transformed + // into writes. Entries are applied in order, meaning that earlier rules will + // affect the results of later ones. + repeated ReadModifyWriteRule rules = 3 [(google.api.field_behavior) = REQUIRED]; +} + +// Response message for Bigtable.ReadModifyWriteRow. +message ReadModifyWriteRowResponse { + // A Row containing the new contents of all cells modified by the request. + Row row = 1; +} diff --git a/proto/google/bigtable/v2/data.proto b/proto/google/bigtable/v2/data.proto new file mode 100644 index 0000000..2cc9164 --- /dev/null +++ b/proto/google/bigtable/v2/data.proto @@ -0,0 +1,536 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.bigtable.v2; + +option csharp_namespace = "Google.Cloud.Bigtable.V2"; +option go_package = "google.golang.org/genproto/googleapis/bigtable/v2;bigtable"; +option java_multiple_files = true; +option java_outer_classname = "DataProto"; +option java_package = "com.google.bigtable.v2"; +option php_namespace = "Google\\Cloud\\Bigtable\\V2"; +option ruby_package = "Google::Cloud::Bigtable::V2"; + +// Specifies the complete (requested) contents of a single row of a table. +// Rows which exceed 256MiB in size cannot be read in full. +message Row { + // The unique key which identifies this row within its table. This is the same + // key that's used to identify the row in, for example, a MutateRowRequest. + // May contain any non-empty byte string up to 4KiB in length. + bytes key = 1; + + // May be empty, but only if the entire row is empty. + // The mutual ordering of column families is not specified. + repeated Family families = 2; +} + +// Specifies (some of) the contents of a single row/column family intersection +// of a table. +message Family { + // The unique key which identifies this family within its row. This is the + // same key that's used to identify the family in, for example, a RowFilter + // which sets its "family_name_regex_filter" field. + // Must match `[-_.a-zA-Z0-9]+`, except that AggregatingRowProcessors may + // produce cells in a sentinel family with an empty name. + // Must be no greater than 64 characters in length. + string name = 1; + + // Must not be empty. Sorted in order of increasing "qualifier". + repeated Column columns = 2; +} + +// Specifies (some of) the contents of a single row/column intersection of a +// table. +message Column { + // The unique key which identifies this column within its family. This is the + // same key that's used to identify the column in, for example, a RowFilter + // which sets its `column_qualifier_regex_filter` field. + // May contain any byte string, including the empty string, up to 16kiB in + // length. + bytes qualifier = 1; + + // Must not be empty. Sorted in order of decreasing "timestamp_micros". + repeated Cell cells = 2; +} + +// Specifies (some of) the contents of a single row/column/timestamp of a table. +message Cell { + // The cell's stored timestamp, which also uniquely identifies it within + // its column. + // Values are always expressed in microseconds, but individual tables may set + // a coarser granularity to further restrict the allowed values. For + // example, a table which specifies millisecond granularity will only allow + // values of `timestamp_micros` which are multiples of 1000. + int64 timestamp_micros = 1; + + // The value stored in the cell. + // May contain any byte string, including the empty string, up to 100MiB in + // length. + bytes value = 2; + + // Labels applied to the cell by a [RowFilter][google.bigtable.v2.RowFilter]. + repeated string labels = 3; +} + +// Specifies a contiguous range of rows. +message RowRange { + // The row key at which to start the range. + // If neither field is set, interpreted as the empty string, inclusive. + oneof start_key { + // Used when giving an inclusive lower bound for the range. + bytes start_key_closed = 1; + + // Used when giving an exclusive lower bound for the range. + bytes start_key_open = 2; + } + + // The row key at which to end the range. + // If neither field is set, interpreted as the infinite row key, exclusive. + oneof end_key { + // Used when giving an exclusive upper bound for the range. + bytes end_key_open = 3; + + // Used when giving an inclusive upper bound for the range. + bytes end_key_closed = 4; + } +} + +// Specifies a non-contiguous set of rows. +message RowSet { + // Single rows included in the set. + repeated bytes row_keys = 1; + + // Contiguous row ranges included in the set. + repeated RowRange row_ranges = 2; +} + +// Specifies a contiguous range of columns within a single column family. +// The range spans from <column_family>:<start_qualifier> to +// <column_family>:<end_qualifier>, where both bounds can be either +// inclusive or exclusive. +message ColumnRange { + // The name of the column family within which this range falls. + string family_name = 1; + + // The column qualifier at which to start the range (within `column_family`). + // If neither field is set, interpreted as the empty string, inclusive. + oneof start_qualifier { + // Used when giving an inclusive lower bound for the range. + bytes start_qualifier_closed = 2; + + // Used when giving an exclusive lower bound for the range. + bytes start_qualifier_open = 3; + } + + // The column qualifier at which to end the range (within `column_family`). + // If neither field is set, interpreted as the infinite string, exclusive. + oneof end_qualifier { + // Used when giving an inclusive upper bound for the range. + bytes end_qualifier_closed = 4; + + // Used when giving an exclusive upper bound for the range. + bytes end_qualifier_open = 5; + } +} + +// Specified a contiguous range of microsecond timestamps. +message TimestampRange { + // Inclusive lower bound. If left empty, interpreted as 0. + int64 start_timestamp_micros = 1; + + // Exclusive upper bound. If left empty, interpreted as infinity. + int64 end_timestamp_micros = 2; +} + +// Specifies a contiguous range of raw byte values. +message ValueRange { + // The value at which to start the range. + // If neither field is set, interpreted as the empty string, inclusive. + oneof start_value { + // Used when giving an inclusive lower bound for the range. + bytes start_value_closed = 1; + + // Used when giving an exclusive lower bound for the range. + bytes start_value_open = 2; + } + + // The value at which to end the range. + // If neither field is set, interpreted as the infinite string, exclusive. + oneof end_value { + // Used when giving an inclusive upper bound for the range. + bytes end_value_closed = 3; + + // Used when giving an exclusive upper bound for the range. + bytes end_value_open = 4; + } +} + +// Takes a row as input and produces an alternate view of the row based on +// specified rules. For example, a RowFilter might trim down a row to include +// just the cells from columns matching a given regular expression, or might +// return all the cells of a row but not their values. More complicated filters +// can be composed out of these components to express requests such as, "within +// every column of a particular family, give just the two most recent cells +// which are older than timestamp X." +// +// There are two broad categories of RowFilters (true filters and transformers), +// as well as two ways to compose simple filters into more complex ones +// (chains and interleaves). They work as follows: +// +// * True filters alter the input row by excluding some of its cells wholesale +// from the output row. An example of a true filter is the `value_regex_filter`, +// which excludes cells whose values don't match the specified pattern. All +// regex true filters use RE2 syntax (https://github.com/google/re2/wiki/Syntax) +// in raw byte mode (RE2::Latin1), and are evaluated as full matches. An +// important point to keep in mind is that `RE2(.)` is equivalent by default to +// `RE2([^\n])`, meaning that it does not match newlines. When attempting to +// match an arbitrary byte, you should therefore use the escape sequence `\C`, +// which may need to be further escaped as `\\C` in your client language. +// +// * Transformers alter the input row by changing the values of some of its +// cells in the output, without excluding them completely. Currently, the only +// supported transformer is the `strip_value_transformer`, which replaces every +// cell's value with the empty string. +// +// * Chains and interleaves are described in more detail in the +// RowFilter.Chain and RowFilter.Interleave documentation. +// +// The total serialized size of a RowFilter message must not +// exceed 4096 bytes, and RowFilters may not be nested within each other +// (in Chains or Interleaves) to a depth of more than 20. +message RowFilter { + // A RowFilter which sends rows through several RowFilters in sequence. + message Chain { + // The elements of "filters" are chained together to process the input row: + // in row -> f(0) -> intermediate row -> f(1) -> ... -> f(N) -> out row + // The full chain is executed atomically. + repeated RowFilter filters = 1; + } + + // A RowFilter which sends each row to each of several component + // RowFilters and interleaves the results. + message Interleave { + // The elements of "filters" all process a copy of the input row, and the + // results are pooled, sorted, and combined into a single output row. + // If multiple cells are produced with the same column and timestamp, + // they will all appear in the output row in an unspecified mutual order. + // Consider the following example, with three filters: + // + // input row + // | + // ----------------------------------------------------- + // | | | + // f(0) f(1) f(2) + // | | | + // 1: foo,bar,10,x foo,bar,10,z far,bar,7,a + // 2: foo,blah,11,z far,blah,5,x far,blah,5,x + // | | | + // ----------------------------------------------------- + // | + // 1: foo,bar,10,z // could have switched with #2 + // 2: foo,bar,10,x // could have switched with #1 + // 3: foo,blah,11,z + // 4: far,bar,7,a + // 5: far,blah,5,x // identical to #6 + // 6: far,blah,5,x // identical to #5 + // + // All interleaved filters are executed atomically. + repeated RowFilter filters = 1; + } + + // A RowFilter which evaluates one of two possible RowFilters, depending on + // whether or not a predicate RowFilter outputs any cells from the input row. + // + // IMPORTANT NOTE: The predicate filter does not execute atomically with the + // true and false filters, which may lead to inconsistent or unexpected + // results. Additionally, Condition filters have poor performance, especially + // when filters are set for the false condition. + message Condition { + // If `predicate_filter` outputs any cells, then `true_filter` will be + // evaluated on the input row. Otherwise, `false_filter` will be evaluated. + RowFilter predicate_filter = 1; + + // The filter to apply to the input row if `predicate_filter` returns any + // results. If not provided, no results will be returned in the true case. + RowFilter true_filter = 2; + + // The filter to apply to the input row if `predicate_filter` does not + // return any results. If not provided, no results will be returned in the + // false case. + RowFilter false_filter = 3; + } + + // Which of the possible RowFilter types to apply. If none are set, this + // RowFilter returns all cells in the input row. + oneof filter { + // Applies several RowFilters to the data in sequence, progressively + // narrowing the results. + Chain chain = 1; + + // Applies several RowFilters to the data in parallel and combines the + // results. + Interleave interleave = 2; + + // Applies one of two possible RowFilters to the data based on the output of + // a predicate RowFilter. + Condition condition = 3; + + // ADVANCED USE ONLY. + // Hook for introspection into the RowFilter. Outputs all cells directly to + // the output of the read rather than to any parent filter. Consider the + // following example: + // + // Chain( + // FamilyRegex("A"), + // Interleave( + // All(), + // Chain(Label("foo"), Sink()) + // ), + // QualifierRegex("B") + // ) + // + // A,A,1,w + // A,B,2,x + // B,B,4,z + // | + // FamilyRegex("A") + // | + // A,A,1,w + // A,B,2,x + // | + // +------------+-------------+ + // | | + // All() Label(foo) + // | | + // A,A,1,w A,A,1,w,labels:[foo] + // A,B,2,x A,B,2,x,labels:[foo] + // | | + // | Sink() --------------+ + // | | | + // +------------+ x------+ A,A,1,w,labels:[foo] + // | A,B,2,x,labels:[foo] + // A,A,1,w | + // A,B,2,x | + // | | + // QualifierRegex("B") | + // | | + // A,B,2,x | + // | | + // +--------------------------------+ + // | + // A,A,1,w,labels:[foo] + // A,B,2,x,labels:[foo] // could be switched + // A,B,2,x // could be switched + // + // Despite being excluded by the qualifier filter, a copy of every cell + // that reaches the sink is present in the final result. + // + // As with an [Interleave][google.bigtable.v2.RowFilter.Interleave], + // duplicate cells are possible, and appear in an unspecified mutual order. + // In this case we have a duplicate with column "A:B" and timestamp 2, + // because one copy passed through the all filter while the other was + // passed through the label and sink. Note that one copy has label "foo", + // while the other does not. + // + // Cannot be used within the `predicate_filter`, `true_filter`, or + // `false_filter` of a [Condition][google.bigtable.v2.RowFilter.Condition]. + bool sink = 16; + + // Matches all cells, regardless of input. Functionally equivalent to + // leaving `filter` unset, but included for completeness. + bool pass_all_filter = 17; + + // Does not match any cells, regardless of input. Useful for temporarily + // disabling just part of a filter. + bool block_all_filter = 18; + + // Matches only cells from rows whose keys satisfy the given RE2 regex. In + // other words, passes through the entire row when the key matches, and + // otherwise produces an empty row. + // Note that, since row keys can contain arbitrary bytes, the `\C` escape + // sequence must be used if a true wildcard is desired. The `.` character + // will not match the new line character `\n`, which may be present in a + // binary key. + bytes row_key_regex_filter = 4; + + // Matches all cells from a row with probability p, and matches no cells + // from the row with probability 1-p. + double row_sample_filter = 14; + + // Matches only cells from columns whose families satisfy the given RE2 + // regex. For technical reasons, the regex must not contain the `:` + // character, even if it is not being used as a literal. + // Note that, since column families cannot contain the new line character + // `\n`, it is sufficient to use `.` as a full wildcard when matching + // column family names. + string family_name_regex_filter = 5; + + // Matches only cells from columns whose qualifiers satisfy the given RE2 + // regex. + // Note that, since column qualifiers can contain arbitrary bytes, the `\C` + // escape sequence must be used if a true wildcard is desired. The `.` + // character will not match the new line character `\n`, which may be + // present in a binary qualifier. + bytes column_qualifier_regex_filter = 6; + + // Matches only cells from columns within the given range. + ColumnRange column_range_filter = 7; + + // Matches only cells with timestamps within the given range. + TimestampRange timestamp_range_filter = 8; + + // Matches only cells with values that satisfy the given regular expression. + // Note that, since cell values can contain arbitrary bytes, the `\C` escape + // sequence must be used if a true wildcard is desired. The `.` character + // will not match the new line character `\n`, which may be present in a + // binary value. + bytes value_regex_filter = 9; + + // Matches only cells with values that fall within the given range. + ValueRange value_range_filter = 15; + + // Skips the first N cells of each row, matching all subsequent cells. + // If duplicate cells are present, as is possible when using an Interleave, + // each copy of the cell is counted separately. + int32 cells_per_row_offset_filter = 10; + + // Matches only the first N cells of each row. + // If duplicate cells are present, as is possible when using an Interleave, + // each copy of the cell is counted separately. + int32 cells_per_row_limit_filter = 11; + + // Matches only the most recent N cells within each column. For example, + // if N=2, this filter would match column `foo:bar` at timestamps 10 and 9, + // skip all earlier cells in `foo:bar`, and then begin matching again in + // column `foo:bar2`. + // If duplicate cells are present, as is possible when using an Interleave, + // each copy of the cell is counted separately. + int32 cells_per_column_limit_filter = 12; + + // Replaces each cell's value with the empty string. + bool strip_value_transformer = 13; + + // Applies the given label to all cells in the output row. This allows + // the client to determine which results were produced from which part of + // the filter. + // + // Values must be at most 15 characters in length, and match the RE2 + // pattern `[a-z0-9\\-]+` + // + // Due to a technical limitation, it is not currently possible to apply + // multiple labels to a cell. As a result, a Chain may have no more than + // one sub-filter which contains a `apply_label_transformer`. It is okay for + // an Interleave to contain multiple `apply_label_transformers`, as they + // will be applied to separate copies of the input. This may be relaxed in + // the future. + string apply_label_transformer = 19; + } +} + +// Specifies a particular change to be made to the contents of a row. +message Mutation { + // A Mutation which sets the value of the specified cell. + message SetCell { + // The name of the family into which new data should be written. + // Must match `[-_.a-zA-Z0-9]+` + string family_name = 1; + + // The qualifier of the column into which new data should be written. + // Can be any byte string, including the empty string. + bytes column_qualifier = 2; + + // The timestamp of the cell into which new data should be written. + // Use -1 for current Bigtable server time. + // Otherwise, the client should set this value itself, noting that the + // default value is a timestamp of zero if the field is left unspecified. + // Values must match the granularity of the table (e.g. micros, millis). + int64 timestamp_micros = 3; + + // The value to be written into the specified cell. + bytes value = 4; + } + + // A Mutation which deletes cells from the specified column, optionally + // restricting the deletions to a given timestamp range. + message DeleteFromColumn { + // The name of the family from which cells should be deleted. + // Must match `[-_.a-zA-Z0-9]+` + string family_name = 1; + + // The qualifier of the column from which cells should be deleted. + // Can be any byte string, including the empty string. + bytes column_qualifier = 2; + + // The range of timestamps within which cells should be deleted. + TimestampRange time_range = 3; + } + + // A Mutation which deletes all cells from the specified column family. + message DeleteFromFamily { + // The name of the family from which cells should be deleted. + // Must match `[-_.a-zA-Z0-9]+` + string family_name = 1; + } + + // A Mutation which deletes all cells from the containing row. + message DeleteFromRow { + + } + + // Which of the possible Mutation types to apply. + oneof mutation { + // Set a cell's value. + SetCell set_cell = 1; + + // Deletes cells from a column. + DeleteFromColumn delete_from_column = 2; + + // Deletes cells from a column family. + DeleteFromFamily delete_from_family = 3; + + // Deletes cells from the entire row. + DeleteFromRow delete_from_row = 4; + } +} + +// Specifies an atomic read/modify/write operation on the latest value of the +// specified column. +message ReadModifyWriteRule { + // The name of the family to which the read/modify/write should be applied. + // Must match `[-_.a-zA-Z0-9]+` + string family_name = 1; + + // The qualifier of the column to which the read/modify/write should be + // applied. + // Can be any byte string, including the empty string. + bytes column_qualifier = 2; + + // The rule used to determine the column's new latest value from its current + // latest value. + oneof rule { + // Rule specifying that `append_value` be appended to the existing value. + // If the targeted cell is unset, it will be treated as containing the + // empty string. + bytes append_value = 3; + + // Rule specifying that `increment_amount` be added to the existing value. + // If the targeted cell is unset, it will be treated as containing a zero. + // Otherwise, the targeted cell must contain an 8-byte value (interpreted + // as a 64-bit big-endian signed integer), or the entire request will fail. + int64 increment_amount = 4; + } +} diff --git a/proto/google/cloud/kms/v1/BUILD b/proto/google/cloud/kms/v1/BUILD new file mode 100644 index 0000000..db8f593 --- /dev/null +++ b/proto/google/cloud/kms/v1/BUILD @@ -0,0 +1,16 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "kms_proto", + srcs = [ + "resources.proto", + "service.proto", + ], + strip_import_prefix = "/proto/", + deps = [ + "//proto/google:com_google_protobuf", + "//proto/google:googleapis", + ], +) diff --git a/proto/google/cloud/kms/v1/resources.proto b/proto/google/cloud/kms/v1/resources.proto new file mode 100644 index 0000000..0ae8b37 --- /dev/null +++ b/proto/google/cloud/kms/v1/resources.proto @@ -0,0 +1,908 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.kms.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +option cc_enable_arenas = true; +option csharp_namespace = "Google.Cloud.Kms.V1"; +option go_package = "cloud.google.com/go/kms/apiv1/kmspb;kmspb"; +option java_multiple_files = true; +option java_outer_classname = "KmsResourcesProto"; +option java_package = "com.google.cloud.kms.v1"; +option php_namespace = "Google\\Cloud\\Kms\\V1"; + +// A [KeyRing][google.cloud.kms.v1.KeyRing] is a toplevel logical grouping of +// [CryptoKeys][google.cloud.kms.v1.CryptoKey]. +message KeyRing { + option (google.api.resource) = { + type: "cloudkms.googleapis.com/KeyRing" + pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}" + }; + + // Output only. The resource name for the + // [KeyRing][google.cloud.kms.v1.KeyRing] in the format + // `projects/*/locations/*/keyRings/*`. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time at which this [KeyRing][google.cloud.kms.v1.KeyRing] + // was created. + google.protobuf.Timestamp create_time = 2 + [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// A [CryptoKey][google.cloud.kms.v1.CryptoKey] represents a logical key that +// can be used for cryptographic operations. +// +// A [CryptoKey][google.cloud.kms.v1.CryptoKey] is made up of zero or more +// [versions][google.cloud.kms.v1.CryptoKeyVersion], which represent the actual +// key material used in cryptographic operations. +message CryptoKey { + option (google.api.resource) = { + type: "cloudkms.googleapis.com/CryptoKey" + pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}" + }; + + // [CryptoKeyPurpose][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose] + // describes the cryptographic capabilities of a + // [CryptoKey][google.cloud.kms.v1.CryptoKey]. A given key can only be used + // for the operations allowed by its purpose. For more information, see [Key + // purposes](https://cloud.google.com/kms/docs/algorithms#key_purposes). + enum CryptoKeyPurpose { + // Not specified. + CRYPTO_KEY_PURPOSE_UNSPECIFIED = 0; + + // [CryptoKeys][google.cloud.kms.v1.CryptoKey] with this purpose may be used + // with [Encrypt][google.cloud.kms.v1.KeyManagementService.Encrypt] and + // [Decrypt][google.cloud.kms.v1.KeyManagementService.Decrypt]. + ENCRYPT_DECRYPT = 1; + + // [CryptoKeys][google.cloud.kms.v1.CryptoKey] with this purpose may be used + // with + // [AsymmetricSign][google.cloud.kms.v1.KeyManagementService.AsymmetricSign] + // and + // [GetPublicKey][google.cloud.kms.v1.KeyManagementService.GetPublicKey]. + ASYMMETRIC_SIGN = 5; + + // [CryptoKeys][google.cloud.kms.v1.CryptoKey] with this purpose may be used + // with + // [AsymmetricDecrypt][google.cloud.kms.v1.KeyManagementService.AsymmetricDecrypt] + // and + // [GetPublicKey][google.cloud.kms.v1.KeyManagementService.GetPublicKey]. + ASYMMETRIC_DECRYPT = 6; + + // [CryptoKeys][google.cloud.kms.v1.CryptoKey] with this purpose may be used + // with [RawEncrypt][google.cloud.kms.v1.KeyManagementService.RawEncrypt] + // and [RawDecrypt][google.cloud.kms.v1.KeyManagementService.RawDecrypt]. + // This purpose is meant to be used for interoperable symmetric + // encryption and does not support automatic CryptoKey rotation. + RAW_ENCRYPT_DECRYPT = 7; + + // [CryptoKeys][google.cloud.kms.v1.CryptoKey] with this purpose may be used + // with [MacSign][google.cloud.kms.v1.KeyManagementService.MacSign]. + MAC = 9; + } + + // Output only. The resource name for this + // [CryptoKey][google.cloud.kms.v1.CryptoKey] in the format + // `projects/*/locations/*/keyRings/*/cryptoKeys/*`. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. A copy of the "primary" + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] that will be used + // by [Encrypt][google.cloud.kms.v1.KeyManagementService.Encrypt] when this + // [CryptoKey][google.cloud.kms.v1.CryptoKey] is given in + // [EncryptRequest.name][google.cloud.kms.v1.EncryptRequest.name]. + // + // The [CryptoKey][google.cloud.kms.v1.CryptoKey]'s primary version can be + // updated via + // [UpdateCryptoKeyPrimaryVersion][google.cloud.kms.v1.KeyManagementService.UpdateCryptoKeyPrimaryVersion]. + // + // Keys with [purpose][google.cloud.kms.v1.CryptoKey.purpose] + // [ENCRYPT_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT] + // may have a primary. For other keys, this field will be omitted. + CryptoKeyVersion primary = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Immutable. The immutable purpose of this + // [CryptoKey][google.cloud.kms.v1.CryptoKey]. + CryptoKeyPurpose purpose = 3 [(google.api.field_behavior) = IMMUTABLE]; + + // Output only. The time at which this + // [CryptoKey][google.cloud.kms.v1.CryptoKey] was created. + google.protobuf.Timestamp create_time = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // At [next_rotation_time][google.cloud.kms.v1.CryptoKey.next_rotation_time], + // the Key Management Service will automatically: + // + // 1. Create a new version of this [CryptoKey][google.cloud.kms.v1.CryptoKey]. + // 2. Mark the new version as primary. + // + // Key rotations performed manually via + // [CreateCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.CreateCryptoKeyVersion] + // and + // [UpdateCryptoKeyPrimaryVersion][google.cloud.kms.v1.KeyManagementService.UpdateCryptoKeyPrimaryVersion] + // do not affect + // [next_rotation_time][google.cloud.kms.v1.CryptoKey.next_rotation_time]. + // + // Keys with [purpose][google.cloud.kms.v1.CryptoKey.purpose] + // [ENCRYPT_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT] + // support automatic rotation. For other keys, this field must be omitted. + google.protobuf.Timestamp next_rotation_time = 7; + + // Controls the rate of automatic rotation. + oneof rotation_schedule { + // [next_rotation_time][google.cloud.kms.v1.CryptoKey.next_rotation_time] + // will be advanced by this period when the service automatically rotates a + // key. Must be at least 24 hours and at most 876,000 hours. + // + // If [rotation_period][google.cloud.kms.v1.CryptoKey.rotation_period] is + // set, + // [next_rotation_time][google.cloud.kms.v1.CryptoKey.next_rotation_time] + // must also be set. + // + // Keys with [purpose][google.cloud.kms.v1.CryptoKey.purpose] + // [ENCRYPT_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT] + // support automatic rotation. For other keys, this field must be omitted. + google.protobuf.Duration rotation_period = 8; + } + + // A template describing settings for new + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] instances. The + // properties of new [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] + // instances created by either + // [CreateCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.CreateCryptoKeyVersion] + // or auto-rotation are controlled by this template. + CryptoKeyVersionTemplate version_template = 11; + + // Labels with user-defined metadata. For more information, see + // [Labeling Keys](https://cloud.google.com/kms/docs/labeling-keys). + map labels = 10; + + // Immutable. Whether this key may contain imported versions only. + bool import_only = 13 [(google.api.field_behavior) = IMMUTABLE]; + + // Immutable. The period of time that versions of this key spend in the + // [DESTROY_SCHEDULED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.DESTROY_SCHEDULED] + // state before transitioning to + // [DESTROYED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.DESTROYED]. + // If not specified at creation time, the default duration is 24 hours. + google.protobuf.Duration destroy_scheduled_duration = 14 + [(google.api.field_behavior) = IMMUTABLE]; + + // Immutable. The resource name of the backend environment where the key + // material for all [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion] + // associated with this [CryptoKey][google.cloud.kms.v1.CryptoKey] reside and + // where all related cryptographic operations are performed. Only applicable + // if [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion] have a + // [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] of + // [EXTERNAL_VPC][CryptoKeyVersion.ProtectionLevel.EXTERNAL_VPC], with the + // resource name in the format `projects/*/locations/*/ekmConnections/*`. + // Note, this list is non-exhaustive and may apply to additional + // [ProtectionLevels][google.cloud.kms.v1.ProtectionLevel] in the future. + string crypto_key_backend = 15 [ + (google.api.field_behavior) = IMMUTABLE, + (google.api.resource_reference) = { type: "*" } + ]; +} + +// A [CryptoKeyVersionTemplate][google.cloud.kms.v1.CryptoKeyVersionTemplate] +// specifies the properties to use when creating a new +// [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion], either manually +// with +// [CreateCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.CreateCryptoKeyVersion] +// or automatically as a result of auto-rotation. +message CryptoKeyVersionTemplate { + // [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] to use when creating + // a [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] based on this + // template. Immutable. Defaults to + // [SOFTWARE][google.cloud.kms.v1.ProtectionLevel.SOFTWARE]. + ProtectionLevel protection_level = 1; + + // Required. + // [Algorithm][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionAlgorithm] + // to use when creating a + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] based on this + // template. + // + // For backwards compatibility, GOOGLE_SYMMETRIC_ENCRYPTION is implied if both + // this field is omitted and + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] is + // [ENCRYPT_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT]. + CryptoKeyVersion.CryptoKeyVersionAlgorithm algorithm = 3 + [(google.api.field_behavior) = REQUIRED]; +} + +// Contains an HSM-generated attestation about a key operation. For more +// information, see [Verifying attestations] +// (https://cloud.google.com/kms/docs/attest-key). +message KeyOperationAttestation { + // Attestation formats provided by the HSM. + enum AttestationFormat { + // Not specified. + ATTESTATION_FORMAT_UNSPECIFIED = 0; + + // Cavium HSM attestation compressed with gzip. Note that this format is + // defined by Cavium and subject to change at any time. + // + // See + // https://www.marvell.com/products/security-solutions/nitrox-hs-adapters/software-key-attestation.html. + CAVIUM_V1_COMPRESSED = 3; + + // Cavium HSM attestation V2 compressed with gzip. This is a new format + // introduced in Cavium's version 3.2-08. + CAVIUM_V2_COMPRESSED = 4; + } + + // Certificate chains needed to verify the attestation. + // Certificates in chains are PEM-encoded and are ordered based on + // https://tools.ietf.org/html/rfc5246#section-7.4.2. + message CertificateChains { + // Cavium certificate chain corresponding to the attestation. + repeated string cavium_certs = 1; + + // Google card certificate chain corresponding to the attestation. + repeated string google_card_certs = 2; + + // Google partition certificate chain corresponding to the attestation. + repeated string google_partition_certs = 3; + } + + // Output only. The format of the attestation data. + AttestationFormat format = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The attestation data provided by the HSM when the key + // operation was performed. + bytes content = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The certificate chains needed to validate the attestation + CertificateChains cert_chains = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// A [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] represents an +// individual cryptographic key, and the associated key material. +// +// An +// [ENABLED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.ENABLED] +// version can be used for cryptographic operations. +// +// For security reasons, the raw cryptographic key material represented by a +// [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] can never be viewed +// or exported. It can only be used to encrypt, decrypt, or sign data when an +// authorized user or application invokes Cloud KMS. +message CryptoKeyVersion { + option (google.api.resource) = { + type: "cloudkms.googleapis.com/CryptoKeyVersion" + pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}/cryptoKeyVersions/{crypto_key_version}" + }; + + // The algorithm of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion], indicating what + // parameters must be used for each cryptographic operation. + // + // The + // [GOOGLE_SYMMETRIC_ENCRYPTION][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionAlgorithm.GOOGLE_SYMMETRIC_ENCRYPTION] + // algorithm is usable with + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] + // [ENCRYPT_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT]. + // + // Algorithms beginning with "RSA_SIGN_" are usable with + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] + // [ASYMMETRIC_SIGN][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ASYMMETRIC_SIGN]. + // + // The fields in the name after "RSA_SIGN_" correspond to the following + // parameters: padding algorithm, modulus bit length, and digest algorithm. + // + // For PSS, the salt length used is equal to the length of digest + // algorithm. For example, + // [RSA_SIGN_PSS_2048_SHA256][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionAlgorithm.RSA_SIGN_PSS_2048_SHA256] + // will use PSS with a salt length of 256 bits or 32 bytes. + // + // Algorithms beginning with "RSA_DECRYPT_" are usable with + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] + // [ASYMMETRIC_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ASYMMETRIC_DECRYPT]. + // + // The fields in the name after "RSA_DECRYPT_" correspond to the following + // parameters: padding algorithm, modulus bit length, and digest algorithm. + // + // Algorithms beginning with "EC_SIGN_" are usable with + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] + // [ASYMMETRIC_SIGN][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ASYMMETRIC_SIGN]. + // + // The fields in the name after "EC_SIGN_" correspond to the following + // parameters: elliptic curve, digest algorithm. + // + // Algorithms beginning with "HMAC_" are usable with + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] + // [MAC][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.MAC]. + // + // The suffix following "HMAC_" corresponds to the hash algorithm being used + // (eg. SHA256). + // + // For more information, see [Key purposes and algorithms] + // (https://cloud.google.com/kms/docs/algorithms). + enum CryptoKeyVersionAlgorithm { + // Not specified. + CRYPTO_KEY_VERSION_ALGORITHM_UNSPECIFIED = 0; + + // Creates symmetric encryption keys. + GOOGLE_SYMMETRIC_ENCRYPTION = 1; + + // AES-GCM (Galois Counter Mode) using 128-bit keys. + AES_128_GCM = 41; + + // AES-GCM (Galois Counter Mode) using 256-bit keys. + AES_256_GCM = 19; + + // RSASSA-PSS 2048 bit key with a SHA256 digest. + RSA_SIGN_PSS_2048_SHA256 = 2; + + // RSASSA-PSS 3072 bit key with a SHA256 digest. + RSA_SIGN_PSS_3072_SHA256 = 3; + + // RSASSA-PSS 4096 bit key with a SHA256 digest. + RSA_SIGN_PSS_4096_SHA256 = 4; + + // RSASSA-PSS 4096 bit key with a SHA512 digest. + RSA_SIGN_PSS_4096_SHA512 = 15; + + // RSASSA-PKCS1-v1_5 with a 2048 bit key and a SHA256 digest. + RSA_SIGN_PKCS1_2048_SHA256 = 5; + + // RSASSA-PKCS1-v1_5 with a 3072 bit key and a SHA256 digest. + RSA_SIGN_PKCS1_3072_SHA256 = 6; + + // RSASSA-PKCS1-v1_5 with a 4096 bit key and a SHA256 digest. + RSA_SIGN_PKCS1_4096_SHA256 = 7; + + // RSASSA-PKCS1-v1_5 with a 4096 bit key and a SHA512 digest. + RSA_SIGN_PKCS1_4096_SHA512 = 16; + + // RSASSA-PKCS1-v1_5 signing without encoding, with a 2048 bit key. + RSA_SIGN_RAW_PKCS1_2048 = 28; + + // RSASSA-PKCS1-v1_5 signing without encoding, with a 3072 bit key. + RSA_SIGN_RAW_PKCS1_3072 = 29; + + // RSASSA-PKCS1-v1_5 signing without encoding, with a 4096 bit key. + RSA_SIGN_RAW_PKCS1_4096 = 30; + + // RSAES-OAEP 2048 bit key with a SHA256 digest. + RSA_DECRYPT_OAEP_2048_SHA256 = 8; + + // RSAES-OAEP 3072 bit key with a SHA256 digest. + RSA_DECRYPT_OAEP_3072_SHA256 = 9; + + // RSAES-OAEP 4096 bit key with a SHA256 digest. + RSA_DECRYPT_OAEP_4096_SHA256 = 10; + + // RSAES-OAEP 4096 bit key with a SHA512 digest. + RSA_DECRYPT_OAEP_4096_SHA512 = 17; + + // RSAES-OAEP 2048 bit key with a SHA1 digest. + RSA_DECRYPT_OAEP_2048_SHA1 = 37; + + // RSAES-OAEP 3072 bit key with a SHA1 digest. + RSA_DECRYPT_OAEP_3072_SHA1 = 38; + + // RSAES-OAEP 4096 bit key with a SHA1 digest. + RSA_DECRYPT_OAEP_4096_SHA1 = 39; + + // ECDSA on the NIST P-256 curve with a SHA256 digest. + // Other hash functions can also be used: + // https://cloud.google.com/kms/docs/create-validate-signatures#ecdsa_support_for_other_hash_algorithms + EC_SIGN_P256_SHA256 = 12; + + // ECDSA on the NIST P-384 curve with a SHA384 digest. + // Other hash functions can also be used: + // https://cloud.google.com/kms/docs/create-validate-signatures#ecdsa_support_for_other_hash_algorithms + EC_SIGN_P384_SHA384 = 13; + + // ECDSA on the non-NIST secp256k1 curve. This curve is only supported for + // HSM protection level. + // Other hash functions can also be used: + // https://cloud.google.com/kms/docs/create-validate-signatures#ecdsa_support_for_other_hash_algorithms + EC_SIGN_SECP256K1_SHA256 = 31; + + // HMAC-SHA256 signing with a 256 bit key. + HMAC_SHA256 = 32; + + // HMAC-SHA1 signing with a 160 bit key. + HMAC_SHA1 = 33; + + // HMAC-SHA384 signing with a 384 bit key. + HMAC_SHA384 = 34; + + // HMAC-SHA512 signing with a 512 bit key. + HMAC_SHA512 = 35; + + // HMAC-SHA224 signing with a 224 bit key. + HMAC_SHA224 = 36; + + // Algorithm representing symmetric encryption by an external key manager. + EXTERNAL_SYMMETRIC_ENCRYPTION = 18; + } + + // The state of a [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion], + // indicating if it can be used. + enum CryptoKeyVersionState { + // Not specified. + CRYPTO_KEY_VERSION_STATE_UNSPECIFIED = 0; + + // This version is still being generated. It may not be used, enabled, + // disabled, or destroyed yet. Cloud KMS will automatically mark this + // version + // [ENABLED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.ENABLED] + // as soon as the version is ready. + PENDING_GENERATION = 5; + + // This version may be used for cryptographic operations. + ENABLED = 1; + + // This version may not be used, but the key material is still available, + // and the version can be placed back into the + // [ENABLED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.ENABLED] + // state. + DISABLED = 2; + + // This version is destroyed, and the key material is no longer stored. + // This version may only become + // [ENABLED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.ENABLED] + // again if this version is + // [reimport_eligible][google.cloud.kms.v1.CryptoKeyVersion.reimport_eligible] + // and the original key material is reimported with a call to + // [KeyManagementService.ImportCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.ImportCryptoKeyVersion]. + DESTROYED = 3; + + // This version is scheduled for destruction, and will be destroyed soon. + // Call + // [RestoreCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.RestoreCryptoKeyVersion] + // to put it back into the + // [DISABLED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.DISABLED] + // state. + DESTROY_SCHEDULED = 4; + + // This version is still being imported. It may not be used, enabled, + // disabled, or destroyed yet. Cloud KMS will automatically mark this + // version + // [ENABLED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.ENABLED] + // as soon as the version is ready. + PENDING_IMPORT = 6; + + // This version was not imported successfully. It may not be used, enabled, + // disabled, or destroyed. The submitted key material has been discarded. + // Additional details can be found in + // [CryptoKeyVersion.import_failure_reason][google.cloud.kms.v1.CryptoKeyVersion.import_failure_reason]. + IMPORT_FAILED = 7; + + // This version was not generated successfully. It may not be used, enabled, + // disabled, or destroyed. Additional details can be found in + // [CryptoKeyVersion.generation_failure_reason][google.cloud.kms.v1.CryptoKeyVersion.generation_failure_reason]. + GENERATION_FAILED = 8; + + // This version was destroyed, and it may not be used or enabled again. + // Cloud KMS is waiting for the corresponding key material residing in an + // external key manager to be destroyed. + PENDING_EXTERNAL_DESTRUCTION = 9; + + // This version was destroyed, and it may not be used or enabled again. + // However, Cloud KMS could not confirm that the corresponding key material + // residing in an external key manager was destroyed. Additional details can + // be found in + // [CryptoKeyVersion.external_destruction_failure_reason][google.cloud.kms.v1.CryptoKeyVersion.external_destruction_failure_reason]. + EXTERNAL_DESTRUCTION_FAILED = 10; + } + + // A view for [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]s. + // Controls the level of detail returned for + // [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion] in + // [KeyManagementService.ListCryptoKeyVersions][google.cloud.kms.v1.KeyManagementService.ListCryptoKeyVersions] + // and + // [KeyManagementService.ListCryptoKeys][google.cloud.kms.v1.KeyManagementService.ListCryptoKeys]. + enum CryptoKeyVersionView { + // Default view for each + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]. Does not + // include the + // [attestation][google.cloud.kms.v1.CryptoKeyVersion.attestation] field. + CRYPTO_KEY_VERSION_VIEW_UNSPECIFIED = 0; + + // Provides all fields in each + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion], including the + // [attestation][google.cloud.kms.v1.CryptoKeyVersion.attestation]. + FULL = 1; + } + + // Output only. The resource name for this + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] in the format + // `projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*`. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // The current state of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]. + CryptoKeyVersionState state = 3; + + // Output only. The [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] + // describing how crypto operations are performed with this + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]. + ProtectionLevel protection_level = 7 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The + // [CryptoKeyVersionAlgorithm][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionAlgorithm] + // that this [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] + // supports. + CryptoKeyVersionAlgorithm algorithm = 10 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Statement that was generated and signed by the HSM at key + // creation time. Use this statement to verify attributes of the key as stored + // on the HSM, independently of Google. Only provided for key versions with + // [protection_level][google.cloud.kms.v1.CryptoKeyVersion.protection_level] + // [HSM][google.cloud.kms.v1.ProtectionLevel.HSM]. + KeyOperationAttestation attestation = 8 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time at which this + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] was created. + google.protobuf.Timestamp create_time = 4 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time this + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]'s key material was + // generated. + google.protobuf.Timestamp generate_time = 11 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time this + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]'s key material is + // scheduled for destruction. Only present if + // [state][google.cloud.kms.v1.CryptoKeyVersion.state] is + // [DESTROY_SCHEDULED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.DESTROY_SCHEDULED]. + google.protobuf.Timestamp destroy_time = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time this CryptoKeyVersion's key material was + // destroyed. Only present if + // [state][google.cloud.kms.v1.CryptoKeyVersion.state] is + // [DESTROYED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.DESTROYED]. + google.protobuf.Timestamp destroy_event_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The name of the [ImportJob][google.cloud.kms.v1.ImportJob] + // used in the most recent import of this + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]. Only present if + // the underlying key material was imported. + string import_job = 14 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time at which this + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]'s key material was + // most recently imported. + google.protobuf.Timestamp import_time = 15 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The root cause of the most recent import failure. Only present + // if [state][google.cloud.kms.v1.CryptoKeyVersion.state] is + // [IMPORT_FAILED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.IMPORT_FAILED]. + string import_failure_reason = 16 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The root cause of the most recent generation failure. Only + // present if [state][google.cloud.kms.v1.CryptoKeyVersion.state] is + // [GENERATION_FAILED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.GENERATION_FAILED]. + string generation_failure_reason = 19 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The root cause of the most recent external destruction + // failure. Only present if + // [state][google.cloud.kms.v1.CryptoKeyVersion.state] is + // [EXTERNAL_DESTRUCTION_FAILED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.EXTERNAL_DESTRUCTION_FAILED]. + string external_destruction_failure_reason = 20 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // ExternalProtectionLevelOptions stores a group of additional fields for + // configuring a [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] that + // are specific to the + // [EXTERNAL][google.cloud.kms.v1.ProtectionLevel.EXTERNAL] protection level + // and [EXTERNAL_VPC][google.cloud.kms.v1.ProtectionLevel.EXTERNAL_VPC] + // protection levels. + ExternalProtectionLevelOptions external_protection_level_options = 17; + + // Output only. Whether or not this key version is eligible for reimport, by + // being specified as a target in + // [ImportCryptoKeyVersionRequest.crypto_key_version][google.cloud.kms.v1.ImportCryptoKeyVersionRequest.crypto_key_version]. + bool reimport_eligible = 18 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// The public key for a given +// [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]. Obtained via +// [GetPublicKey][google.cloud.kms.v1.KeyManagementService.GetPublicKey]. +message PublicKey { + option (google.api.resource) = { + type: "cloudkms.googleapis.com/PublicKey" + pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}/cryptoKeyVersions/{crypto_key_version}/publicKey" + }; + + // The public key, encoded in PEM format. For more information, see the + // [RFC 7468](https://tools.ietf.org/html/rfc7468) sections for + // [General Considerations](https://tools.ietf.org/html/rfc7468#section-2) and + // [Textual Encoding of Subject Public Key Info] + // (https://tools.ietf.org/html/rfc7468#section-13). + string pem = 1; + + // The + // [Algorithm][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionAlgorithm] + // associated with this key. + CryptoKeyVersion.CryptoKeyVersionAlgorithm algorithm = 2; + + // Integrity verification field. A CRC32C checksum of the returned + // [PublicKey.pem][google.cloud.kms.v1.PublicKey.pem]. An integrity check of + // [PublicKey.pem][google.cloud.kms.v1.PublicKey.pem] can be performed by + // computing the CRC32C checksum of + // [PublicKey.pem][google.cloud.kms.v1.PublicKey.pem] and comparing your + // results to this field. Discard the response in case of non-matching + // checksum values, and perform a limited number of retries. A persistent + // mismatch may indicate an issue in your computation of the CRC32C checksum. + // Note: This field is defined as int64 for reasons of compatibility across + // different languages. However, it is a non-negative integer, which will + // never exceed 2^32-1, and can be safely downconverted to uint32 in languages + // that support this type. + // + // NOTE: This field is in Beta. + google.protobuf.Int64Value pem_crc32c = 3; + + // The [name][google.cloud.kms.v1.CryptoKeyVersion.name] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] public key. + // Provided here for verification. + // + // NOTE: This field is in Beta. + string name = 4; + + // The [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] public key. + ProtectionLevel protection_level = 5; +} + +// An [ImportJob][google.cloud.kms.v1.ImportJob] can be used to create +// [CryptoKeys][google.cloud.kms.v1.CryptoKey] and +// [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion] using pre-existing +// key material, generated outside of Cloud KMS. +// +// When an [ImportJob][google.cloud.kms.v1.ImportJob] is created, Cloud KMS will +// generate a "wrapping key", which is a public/private key pair. You use the +// wrapping key to encrypt (also known as wrap) the pre-existing key material to +// protect it during the import process. The nature of the wrapping key depends +// on the choice of +// [import_method][google.cloud.kms.v1.ImportJob.import_method]. When the +// wrapping key generation is complete, the +// [state][google.cloud.kms.v1.ImportJob.state] will be set to +// [ACTIVE][google.cloud.kms.v1.ImportJob.ImportJobState.ACTIVE] and the +// [public_key][google.cloud.kms.v1.ImportJob.public_key] can be fetched. The +// fetched public key can then be used to wrap your pre-existing key material. +// +// Once the key material is wrapped, it can be imported into a new +// [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] in an existing +// [CryptoKey][google.cloud.kms.v1.CryptoKey] by calling +// [ImportCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.ImportCryptoKeyVersion]. +// Multiple [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion] can be +// imported with a single [ImportJob][google.cloud.kms.v1.ImportJob]. Cloud KMS +// uses the private key portion of the wrapping key to unwrap the key material. +// Only Cloud KMS has access to the private key. +// +// An [ImportJob][google.cloud.kms.v1.ImportJob] expires 3 days after it is +// created. Once expired, Cloud KMS will no longer be able to import or unwrap +// any key material that was wrapped with the +// [ImportJob][google.cloud.kms.v1.ImportJob]'s public key. +// +// For more information, see +// [Importing a key](https://cloud.google.com/kms/docs/importing-a-key). +message ImportJob { + option (google.api.resource) = { + type: "cloudkms.googleapis.com/ImportJob" + pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/importJobs/{import_job}" + }; + + // [ImportMethod][google.cloud.kms.v1.ImportJob.ImportMethod] describes the + // key wrapping method chosen for this + // [ImportJob][google.cloud.kms.v1.ImportJob]. + enum ImportMethod { + // Not specified. + IMPORT_METHOD_UNSPECIFIED = 0; + + // This ImportMethod represents the CKM_RSA_AES_KEY_WRAP key wrapping + // scheme defined in the PKCS #11 standard. In summary, this involves + // wrapping the raw key with an ephemeral AES key, and wrapping the + // ephemeral AES key with a 3072 bit RSA key. For more details, see + // [RSA AES key wrap + // mechanism](http://docs.oasis-open.org/pkcs11/pkcs11-curr/v2.40/cos01/pkcs11-curr-v2.40-cos01.html#_Toc408226908). + RSA_OAEP_3072_SHA1_AES_256 = 1; + + // This ImportMethod represents the CKM_RSA_AES_KEY_WRAP key wrapping + // scheme defined in the PKCS #11 standard. In summary, this involves + // wrapping the raw key with an ephemeral AES key, and wrapping the + // ephemeral AES key with a 4096 bit RSA key. For more details, see + // [RSA AES key wrap + // mechanism](http://docs.oasis-open.org/pkcs11/pkcs11-curr/v2.40/cos01/pkcs11-curr-v2.40-cos01.html#_Toc408226908). + RSA_OAEP_4096_SHA1_AES_256 = 2; + + // This ImportMethod represents the CKM_RSA_AES_KEY_WRAP key wrapping + // scheme defined in the PKCS #11 standard. In summary, this involves + // wrapping the raw key with an ephemeral AES key, and wrapping the + // ephemeral AES key with a 3072 bit RSA key. For more details, see + // [RSA AES key wrap + // mechanism](http://docs.oasis-open.org/pkcs11/pkcs11-curr/v2.40/cos01/pkcs11-curr-v2.40-cos01.html#_Toc408226908). + RSA_OAEP_3072_SHA256_AES_256 = 3; + + // This ImportMethod represents the CKM_RSA_AES_KEY_WRAP key wrapping + // scheme defined in the PKCS #11 standard. In summary, this involves + // wrapping the raw key with an ephemeral AES key, and wrapping the + // ephemeral AES key with a 4096 bit RSA key. For more details, see + // [RSA AES key wrap + // mechanism](http://docs.oasis-open.org/pkcs11/pkcs11-curr/v2.40/cos01/pkcs11-curr-v2.40-cos01.html#_Toc408226908). + RSA_OAEP_4096_SHA256_AES_256 = 4; + + // This ImportMethod represents RSAES-OAEP with a 3072 bit RSA key. The + // key material to be imported is wrapped directly with the RSA key. Due + // to technical limitations of RSA wrapping, this method cannot be used to + // wrap RSA keys for import. + RSA_OAEP_3072_SHA256 = 5; + + // This ImportMethod represents RSAES-OAEP with a 4096 bit RSA key. The + // key material to be imported is wrapped directly with the RSA key. Due + // to technical limitations of RSA wrapping, this method cannot be used to + // wrap RSA keys for import. + RSA_OAEP_4096_SHA256 = 6; + } + + // The state of the [ImportJob][google.cloud.kms.v1.ImportJob], indicating if + // it can be used. + enum ImportJobState { + // Not specified. + IMPORT_JOB_STATE_UNSPECIFIED = 0; + + // The wrapping key for this job is still being generated. It may not be + // used. Cloud KMS will automatically mark this job as + // [ACTIVE][google.cloud.kms.v1.ImportJob.ImportJobState.ACTIVE] as soon as + // the wrapping key is generated. + PENDING_GENERATION = 1; + + // This job may be used in + // [CreateCryptoKey][google.cloud.kms.v1.KeyManagementService.CreateCryptoKey] + // and + // [CreateCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.CreateCryptoKeyVersion] + // requests. + ACTIVE = 2; + + // This job can no longer be used and may not leave this state once entered. + EXPIRED = 3; + } + + // The public key component of the wrapping key. For details of the type of + // key this public key corresponds to, see the + // [ImportMethod][google.cloud.kms.v1.ImportJob.ImportMethod]. + message WrappingPublicKey { + // The public key, encoded in PEM format. For more information, see the [RFC + // 7468](https://tools.ietf.org/html/rfc7468) sections for [General + // Considerations](https://tools.ietf.org/html/rfc7468#section-2) and + // [Textual Encoding of Subject Public Key Info] + // (https://tools.ietf.org/html/rfc7468#section-13). + string pem = 1; + } + + // Output only. The resource name for this + // [ImportJob][google.cloud.kms.v1.ImportJob] in the format + // `projects/*/locations/*/keyRings/*/importJobs/*`. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Required. Immutable. The wrapping method to be used for incoming key + // material. + ImportMethod import_method = 2 [ + (google.api.field_behavior) = REQUIRED, + (google.api.field_behavior) = IMMUTABLE + ]; + + // Required. Immutable. The protection level of the + // [ImportJob][google.cloud.kms.v1.ImportJob]. This must match the + // [protection_level][google.cloud.kms.v1.CryptoKeyVersionTemplate.protection_level] + // of the [version_template][google.cloud.kms.v1.CryptoKey.version_template] + // on the [CryptoKey][google.cloud.kms.v1.CryptoKey] you attempt to import + // into. + ProtectionLevel protection_level = 9 [ + (google.api.field_behavior) = REQUIRED, + (google.api.field_behavior) = IMMUTABLE + ]; + + // Output only. The time at which this + // [ImportJob][google.cloud.kms.v1.ImportJob] was created. + google.protobuf.Timestamp create_time = 3 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time this [ImportJob][google.cloud.kms.v1.ImportJob]'s key + // material was generated. + google.protobuf.Timestamp generate_time = 4 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time at which this + // [ImportJob][google.cloud.kms.v1.ImportJob] is scheduled for expiration and + // can no longer be used to import key material. + google.protobuf.Timestamp expire_time = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time this [ImportJob][google.cloud.kms.v1.ImportJob] + // expired. Only present if [state][google.cloud.kms.v1.ImportJob.state] is + // [EXPIRED][google.cloud.kms.v1.ImportJob.ImportJobState.EXPIRED]. + google.protobuf.Timestamp expire_event_time = 10 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The current state of the + // [ImportJob][google.cloud.kms.v1.ImportJob], indicating if it can be used. + ImportJobState state = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The public key with which to wrap key material prior to + // import. Only returned if [state][google.cloud.kms.v1.ImportJob.state] is + // [ACTIVE][google.cloud.kms.v1.ImportJob.ImportJobState.ACTIVE]. + WrappingPublicKey public_key = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Statement that was generated and signed by the key creator + // (for example, an HSM) at key creation time. Use this statement to verify + // attributes of the key as stored on the HSM, independently of Google. + // Only present if the chosen + // [ImportMethod][google.cloud.kms.v1.ImportJob.ImportMethod] is one with a + // protection level of [HSM][google.cloud.kms.v1.ProtectionLevel.HSM]. + KeyOperationAttestation attestation = 8 + [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// ExternalProtectionLevelOptions stores a group of additional fields for +// configuring a [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] that +// are specific to the [EXTERNAL][google.cloud.kms.v1.ProtectionLevel.EXTERNAL] +// protection level and +// [EXTERNAL_VPC][google.cloud.kms.v1.ProtectionLevel.EXTERNAL_VPC] protection +// levels. +message ExternalProtectionLevelOptions { + // The URI for an external resource that this + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] represents. + string external_key_uri = 1; + + // The path to the external key material on the EKM when using + // [EkmConnection][google.cloud.kms.v1.EkmConnection] e.g., "v0/my/key". Set + // this field instead of external_key_uri when using an + // [EkmConnection][google.cloud.kms.v1.EkmConnection]. + string ekm_connection_key_path = 2; +} + +// [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] specifies how +// cryptographic operations are performed. For more information, see [Protection +// levels] (https://cloud.google.com/kms/docs/algorithms#protection_levels). +enum ProtectionLevel { + // Not specified. + PROTECTION_LEVEL_UNSPECIFIED = 0; + + // Crypto operations are performed in software. + SOFTWARE = 1; + + // Crypto operations are performed in a Hardware Security Module. + HSM = 2; + + // Crypto operations are performed by an external key manager. + EXTERNAL = 3; + + // Crypto operations are performed in an EKM-over-VPC backend. + EXTERNAL_VPC = 4; +} diff --git a/proto/google/cloud/kms/v1/service.proto b/proto/google/cloud/kms/v1/service.proto new file mode 100644 index 0000000..c9a0557 --- /dev/null +++ b/proto/google/cloud/kms/v1/service.proto @@ -0,0 +1,2073 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.kms.v1; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/cloud/kms/v1/resources.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/wrappers.proto"; + +option cc_enable_arenas = true; +option csharp_namespace = "Google.Cloud.Kms.V1"; +option go_package = "cloud.google.com/go/kms/apiv1/kmspb;kmspb"; +option java_multiple_files = true; +option java_outer_classname = "KmsProto"; +option java_package = "com.google.cloud.kms.v1"; +option php_namespace = "Google\\Cloud\\Kms\\V1"; + +// Google Cloud Key Management Service +// +// Manages cryptographic keys and operations using those keys. Implements a REST +// model with the following objects: +// +// * [KeyRing][google.cloud.kms.v1.KeyRing] +// * [CryptoKey][google.cloud.kms.v1.CryptoKey] +// * [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] +// * [ImportJob][google.cloud.kms.v1.ImportJob] +// +// If you are using manual gRPC libraries, see +// [Using gRPC with Cloud KMS](https://cloud.google.com/kms/docs/grpc). +service KeyManagementService { + option (google.api.default_host) = "cloudkms.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/cloudkms"; + + // Lists [KeyRings][google.cloud.kms.v1.KeyRing]. + rpc ListKeyRings(ListKeyRingsRequest) returns (ListKeyRingsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*}/keyRings" + }; + option (google.api.method_signature) = "parent"; + } + + // Lists [CryptoKeys][google.cloud.kms.v1.CryptoKey]. + rpc ListCryptoKeys(ListCryptoKeysRequest) returns (ListCryptoKeysResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/keyRings/*}/cryptoKeys" + }; + option (google.api.method_signature) = "parent"; + } + + // Lists [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion]. + rpc ListCryptoKeyVersions(ListCryptoKeyVersionsRequest) + returns (ListCryptoKeyVersionsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/keyRings/*/cryptoKeys/*}/cryptoKeyVersions" + }; + option (google.api.method_signature) = "parent"; + } + + // Lists [ImportJobs][google.cloud.kms.v1.ImportJob]. + rpc ListImportJobs(ListImportJobsRequest) returns (ListImportJobsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/keyRings/*}/importJobs" + }; + option (google.api.method_signature) = "parent"; + } + + // Returns metadata for a given [KeyRing][google.cloud.kms.v1.KeyRing]. + rpc GetKeyRing(GetKeyRingRequest) returns (KeyRing) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/keyRings/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Returns metadata for a given [CryptoKey][google.cloud.kms.v1.CryptoKey], as + // well as its [primary][google.cloud.kms.v1.CryptoKey.primary] + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]. + rpc GetCryptoKey(GetCryptoKeyRequest) returns (CryptoKey) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Returns metadata for a given + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]. + rpc GetCryptoKeyVersion(GetCryptoKeyVersionRequest) + returns (CryptoKeyVersion) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Returns the public key for the given + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]. The + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] must be + // [ASYMMETRIC_SIGN][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ASYMMETRIC_SIGN] + // or + // [ASYMMETRIC_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ASYMMETRIC_DECRYPT]. + rpc GetPublicKey(GetPublicKeyRequest) returns (PublicKey) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*}/publicKey" + }; + option (google.api.method_signature) = "name"; + } + + // Returns metadata for a given [ImportJob][google.cloud.kms.v1.ImportJob]. + rpc GetImportJob(GetImportJobRequest) returns (ImportJob) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/keyRings/*/importJobs/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Create a new [KeyRing][google.cloud.kms.v1.KeyRing] in a given Project and + // Location. + rpc CreateKeyRing(CreateKeyRingRequest) returns (KeyRing) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*}/keyRings" + body: "key_ring" + }; + option (google.api.method_signature) = "parent,key_ring_id,key_ring"; + } + + // Create a new [CryptoKey][google.cloud.kms.v1.CryptoKey] within a + // [KeyRing][google.cloud.kms.v1.KeyRing]. + // + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] and + // [CryptoKey.version_template.algorithm][google.cloud.kms.v1.CryptoKeyVersionTemplate.algorithm] + // are required. + rpc CreateCryptoKey(CreateCryptoKeyRequest) returns (CryptoKey) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/keyRings/*}/cryptoKeys" + body: "crypto_key" + }; + option (google.api.method_signature) = "parent,crypto_key_id,crypto_key"; + } + + // Create a new [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] in a + // [CryptoKey][google.cloud.kms.v1.CryptoKey]. + // + // The server will assign the next sequential id. If unset, + // [state][google.cloud.kms.v1.CryptoKeyVersion.state] will be set to + // [ENABLED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.ENABLED]. + rpc CreateCryptoKeyVersion(CreateCryptoKeyVersionRequest) + returns (CryptoKeyVersion) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/keyRings/*/cryptoKeys/*}/cryptoKeyVersions" + body: "crypto_key_version" + }; + option (google.api.method_signature) = "parent,crypto_key_version"; + } + + // Import wrapped key material into a + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]. + // + // All requests must specify a [CryptoKey][google.cloud.kms.v1.CryptoKey]. If + // a [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] is additionally + // specified in the request, key material will be reimported into that + // version. Otherwise, a new version will be created, and will be assigned the + // next sequential id within the [CryptoKey][google.cloud.kms.v1.CryptoKey]. + rpc ImportCryptoKeyVersion(ImportCryptoKeyVersionRequest) + returns (CryptoKeyVersion) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/keyRings/*/cryptoKeys/*}/cryptoKeyVersions:import" + body: "*" + }; + } + + // Create a new [ImportJob][google.cloud.kms.v1.ImportJob] within a + // [KeyRing][google.cloud.kms.v1.KeyRing]. + // + // [ImportJob.import_method][google.cloud.kms.v1.ImportJob.import_method] is + // required. + rpc CreateImportJob(CreateImportJobRequest) returns (ImportJob) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/keyRings/*}/importJobs" + body: "import_job" + }; + option (google.api.method_signature) = "parent,import_job_id,import_job"; + } + + // Update a [CryptoKey][google.cloud.kms.v1.CryptoKey]. + rpc UpdateCryptoKey(UpdateCryptoKeyRequest) returns (CryptoKey) { + option (google.api.http) = { + patch: "/v1/{crypto_key.name=projects/*/locations/*/keyRings/*/cryptoKeys/*}" + body: "crypto_key" + }; + option (google.api.method_signature) = "crypto_key,update_mask"; + } + + // Update a [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]'s + // metadata. + // + // [state][google.cloud.kms.v1.CryptoKeyVersion.state] may be changed between + // [ENABLED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.ENABLED] + // and + // [DISABLED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.DISABLED] + // using this method. See + // [DestroyCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.DestroyCryptoKeyVersion] + // and + // [RestoreCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.RestoreCryptoKeyVersion] + // to move between other states. + rpc UpdateCryptoKeyVersion(UpdateCryptoKeyVersionRequest) + returns (CryptoKeyVersion) { + option (google.api.http) = { + patch: "/v1/{crypto_key_version.name=projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*}" + body: "crypto_key_version" + }; + option (google.api.method_signature) = "crypto_key_version,update_mask"; + } + + // Update the version of a [CryptoKey][google.cloud.kms.v1.CryptoKey] that + // will be used in + // [Encrypt][google.cloud.kms.v1.KeyManagementService.Encrypt]. + // + // Returns an error if called on a key whose purpose is not + // [ENCRYPT_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT]. + rpc UpdateCryptoKeyPrimaryVersion(UpdateCryptoKeyPrimaryVersionRequest) + returns (CryptoKey) { + option (google.api.http) = { + post: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*}:updatePrimaryVersion" + body: "*" + }; + option (google.api.method_signature) = "name,crypto_key_version_id"; + } + + // Schedule a [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] for + // destruction. + // + // Upon calling this method, + // [CryptoKeyVersion.state][google.cloud.kms.v1.CryptoKeyVersion.state] will + // be set to + // [DESTROY_SCHEDULED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.DESTROY_SCHEDULED], + // and [destroy_time][google.cloud.kms.v1.CryptoKeyVersion.destroy_time] will + // be set to the time + // [destroy_scheduled_duration][google.cloud.kms.v1.CryptoKey.destroy_scheduled_duration] + // in the future. At that time, the + // [state][google.cloud.kms.v1.CryptoKeyVersion.state] will automatically + // change to + // [DESTROYED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.DESTROYED], + // and the key material will be irrevocably destroyed. + // + // Before the + // [destroy_time][google.cloud.kms.v1.CryptoKeyVersion.destroy_time] is + // reached, + // [RestoreCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.RestoreCryptoKeyVersion] + // may be called to reverse the process. + rpc DestroyCryptoKeyVersion(DestroyCryptoKeyVersionRequest) + returns (CryptoKeyVersion) { + option (google.api.http) = { + post: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*}:destroy" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + + // Restore a [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] in the + // [DESTROY_SCHEDULED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.DESTROY_SCHEDULED] + // state. + // + // Upon restoration of the CryptoKeyVersion, + // [state][google.cloud.kms.v1.CryptoKeyVersion.state] will be set to + // [DISABLED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.DISABLED], + // and [destroy_time][google.cloud.kms.v1.CryptoKeyVersion.destroy_time] will + // be cleared. + rpc RestoreCryptoKeyVersion(RestoreCryptoKeyVersionRequest) + returns (CryptoKeyVersion) { + option (google.api.http) = { + post: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*}:restore" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + + // Encrypts data, so that it can only be recovered by a call to + // [Decrypt][google.cloud.kms.v1.KeyManagementService.Decrypt]. The + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] must be + // [ENCRYPT_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT]. + rpc Encrypt(EncryptRequest) returns (EncryptResponse) { + option (google.api.http) = { + post: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/**}:encrypt" + body: "*" + }; + option (google.api.method_signature) = "name,plaintext"; + } + + // Decrypts data that was protected by + // [Encrypt][google.cloud.kms.v1.KeyManagementService.Encrypt]. The + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] must be + // [ENCRYPT_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT]. + rpc Decrypt(DecryptRequest) returns (DecryptResponse) { + option (google.api.http) = { + post: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*}:decrypt" + body: "*" + }; + option (google.api.method_signature) = "name,ciphertext"; + } + + // Encrypts data using portable cryptographic primitives. Most users should + // choose [Encrypt][google.cloud.kms.v1.KeyManagementService.Encrypt] and + // [Decrypt][google.cloud.kms.v1.KeyManagementService.Decrypt] rather than + // their raw counterparts. The + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] must be + // [RAW_ENCRYPT_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.RAW_ENCRYPT_DECRYPT]. + rpc RawEncrypt(RawEncryptRequest) returns (RawEncryptResponse) { + option (google.api.http) = { + post: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*}:rawEncrypt" + body: "*" + }; + } + + // Decrypts data that was originally encrypted using a raw cryptographic + // mechanism. The [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] + // must be + // [RAW_ENCRYPT_DECRYPT][google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.RAW_ENCRYPT_DECRYPT]. + rpc RawDecrypt(RawDecryptRequest) returns (RawDecryptResponse) { + option (google.api.http) = { + post: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*}:rawDecrypt" + body: "*" + }; + } + + // Signs data using a [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] + // with [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] + // ASYMMETRIC_SIGN, producing a signature that can be verified with the public + // key retrieved from + // [GetPublicKey][google.cloud.kms.v1.KeyManagementService.GetPublicKey]. + rpc AsymmetricSign(AsymmetricSignRequest) returns (AsymmetricSignResponse) { + option (google.api.http) = { + post: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*}:asymmetricSign" + body: "*" + }; + option (google.api.method_signature) = "name,digest"; + } + + // Decrypts data that was encrypted with a public key retrieved from + // [GetPublicKey][google.cloud.kms.v1.KeyManagementService.GetPublicKey] + // corresponding to a [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] + // with [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] + // ASYMMETRIC_DECRYPT. + rpc AsymmetricDecrypt(AsymmetricDecryptRequest) + returns (AsymmetricDecryptResponse) { + option (google.api.http) = { + post: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*}:asymmetricDecrypt" + body: "*" + }; + option (google.api.method_signature) = "name,ciphertext"; + } + + // Signs data using a [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] + // with [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] MAC, + // producing a tag that can be verified by another source with the same key. + rpc MacSign(MacSignRequest) returns (MacSignResponse) { + option (google.api.http) = { + post: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*}:macSign" + body: "*" + }; + option (google.api.method_signature) = "name,data"; + } + + // Verifies MAC tag using a + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] with + // [CryptoKey.purpose][google.cloud.kms.v1.CryptoKey.purpose] MAC, and returns + // a response that indicates whether or not the verification was successful. + rpc MacVerify(MacVerifyRequest) returns (MacVerifyResponse) { + option (google.api.http) = { + post: "/v1/{name=projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*}:macVerify" + body: "*" + }; + option (google.api.method_signature) = "name,data,mac"; + } + + // Generate random bytes using the Cloud KMS randomness source in the provided + // location. + rpc GenerateRandomBytes(GenerateRandomBytesRequest) + returns (GenerateRandomBytesResponse) { + option (google.api.http) = { + post: "/v1/{location=projects/*/locations/*}:generateRandomBytes" + body: "*" + }; + option (google.api.method_signature) = + "location,length_bytes,protection_level"; + } +} + +// Request message for +// [KeyManagementService.ListKeyRings][google.cloud.kms.v1.KeyManagementService.ListKeyRings]. +message ListKeyRingsRequest { + // Required. The resource name of the location associated with the + // [KeyRings][google.cloud.kms.v1.KeyRing], in the format + // `projects/*/locations/*`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "locations.googleapis.com/Location" + } + ]; + + // Optional. Optional limit on the number of + // [KeyRings][google.cloud.kms.v1.KeyRing] to include in the response. Further + // [KeyRings][google.cloud.kms.v1.KeyRing] can subsequently be obtained by + // including the + // [ListKeyRingsResponse.next_page_token][google.cloud.kms.v1.ListKeyRingsResponse.next_page_token] + // in a subsequent request. If unspecified, the server will pick an + // appropriate default. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Optional pagination token, returned earlier via + // [ListKeyRingsResponse.next_page_token][google.cloud.kms.v1.ListKeyRingsResponse.next_page_token]. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Only include resources that match the filter in the response. For + // more information, see + // [Sorting and filtering list + // results](https://cloud.google.com/kms/docs/sorting-and-filtering). + string filter = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Specify how the results should be sorted. If not specified, the + // results will be sorted in the default order. For more information, see + // [Sorting and filtering list + // results](https://cloud.google.com/kms/docs/sorting-and-filtering). + string order_by = 5 [(google.api.field_behavior) = OPTIONAL]; +} + +// Request message for +// [KeyManagementService.ListCryptoKeys][google.cloud.kms.v1.KeyManagementService.ListCryptoKeys]. +message ListCryptoKeysRequest { + // Required. The resource name of the [KeyRing][google.cloud.kms.v1.KeyRing] + // to list, in the format `projects/*/locations/*/keyRings/*`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/KeyRing" + } + ]; + + // Optional. Optional limit on the number of + // [CryptoKeys][google.cloud.kms.v1.CryptoKey] to include in the response. + // Further [CryptoKeys][google.cloud.kms.v1.CryptoKey] can subsequently be + // obtained by including the + // [ListCryptoKeysResponse.next_page_token][google.cloud.kms.v1.ListCryptoKeysResponse.next_page_token] + // in a subsequent request. If unspecified, the server will pick an + // appropriate default. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Optional pagination token, returned earlier via + // [ListCryptoKeysResponse.next_page_token][google.cloud.kms.v1.ListCryptoKeysResponse.next_page_token]. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; + + // The fields of the primary version to include in the response. + CryptoKeyVersion.CryptoKeyVersionView version_view = 4; + + // Optional. Only include resources that match the filter in the response. For + // more information, see + // [Sorting and filtering list + // results](https://cloud.google.com/kms/docs/sorting-and-filtering). + string filter = 5 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Specify how the results should be sorted. If not specified, the + // results will be sorted in the default order. For more information, see + // [Sorting and filtering list + // results](https://cloud.google.com/kms/docs/sorting-and-filtering). + string order_by = 6 [(google.api.field_behavior) = OPTIONAL]; +} + +// Request message for +// [KeyManagementService.ListCryptoKeyVersions][google.cloud.kms.v1.KeyManagementService.ListCryptoKeyVersions]. +message ListCryptoKeyVersionsRequest { + // Required. The resource name of the + // [CryptoKey][google.cloud.kms.v1.CryptoKey] to list, in the format + // `projects/*/locations/*/keyRings/*/cryptoKeys/*`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKey" + } + ]; + + // Optional. Optional limit on the number of + // [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion] to include in the + // response. Further [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion] + // can subsequently be obtained by including the + // [ListCryptoKeyVersionsResponse.next_page_token][google.cloud.kms.v1.ListCryptoKeyVersionsResponse.next_page_token] + // in a subsequent request. If unspecified, the server will pick an + // appropriate default. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Optional pagination token, returned earlier via + // [ListCryptoKeyVersionsResponse.next_page_token][google.cloud.kms.v1.ListCryptoKeyVersionsResponse.next_page_token]. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; + + // The fields to include in the response. + CryptoKeyVersion.CryptoKeyVersionView view = 4; + + // Optional. Only include resources that match the filter in the response. For + // more information, see + // [Sorting and filtering list + // results](https://cloud.google.com/kms/docs/sorting-and-filtering). + string filter = 5 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Specify how the results should be sorted. If not specified, the + // results will be sorted in the default order. For more information, see + // [Sorting and filtering list + // results](https://cloud.google.com/kms/docs/sorting-and-filtering). + string order_by = 6 [(google.api.field_behavior) = OPTIONAL]; +} + +// Request message for +// [KeyManagementService.ListImportJobs][google.cloud.kms.v1.KeyManagementService.ListImportJobs]. +message ListImportJobsRequest { + // Required. The resource name of the [KeyRing][google.cloud.kms.v1.KeyRing] + // to list, in the format `projects/*/locations/*/keyRings/*`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/KeyRing" + } + ]; + + // Optional. Optional limit on the number of + // [ImportJobs][google.cloud.kms.v1.ImportJob] to include in the response. + // Further [ImportJobs][google.cloud.kms.v1.ImportJob] can subsequently be + // obtained by including the + // [ListImportJobsResponse.next_page_token][google.cloud.kms.v1.ListImportJobsResponse.next_page_token] + // in a subsequent request. If unspecified, the server will pick an + // appropriate default. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Optional pagination token, returned earlier via + // [ListImportJobsResponse.next_page_token][google.cloud.kms.v1.ListImportJobsResponse.next_page_token]. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Only include resources that match the filter in the response. For + // more information, see + // [Sorting and filtering list + // results](https://cloud.google.com/kms/docs/sorting-and-filtering). + string filter = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Specify how the results should be sorted. If not specified, the + // results will be sorted in the default order. For more information, see + // [Sorting and filtering list + // results](https://cloud.google.com/kms/docs/sorting-and-filtering). + string order_by = 5 [(google.api.field_behavior) = OPTIONAL]; +} + +// Response message for +// [KeyManagementService.ListKeyRings][google.cloud.kms.v1.KeyManagementService.ListKeyRings]. +message ListKeyRingsResponse { + // The list of [KeyRings][google.cloud.kms.v1.KeyRing]. + repeated KeyRing key_rings = 1; + + // A token to retrieve next page of results. Pass this value in + // [ListKeyRingsRequest.page_token][google.cloud.kms.v1.ListKeyRingsRequest.page_token] + // to retrieve the next page of results. + string next_page_token = 2; + + // The total number of [KeyRings][google.cloud.kms.v1.KeyRing] that matched + // the query. + int32 total_size = 3; +} + +// Response message for +// [KeyManagementService.ListCryptoKeys][google.cloud.kms.v1.KeyManagementService.ListCryptoKeys]. +message ListCryptoKeysResponse { + // The list of [CryptoKeys][google.cloud.kms.v1.CryptoKey]. + repeated CryptoKey crypto_keys = 1; + + // A token to retrieve next page of results. Pass this value in + // [ListCryptoKeysRequest.page_token][google.cloud.kms.v1.ListCryptoKeysRequest.page_token] + // to retrieve the next page of results. + string next_page_token = 2; + + // The total number of [CryptoKeys][google.cloud.kms.v1.CryptoKey] that + // matched the query. + int32 total_size = 3; +} + +// Response message for +// [KeyManagementService.ListCryptoKeyVersions][google.cloud.kms.v1.KeyManagementService.ListCryptoKeyVersions]. +message ListCryptoKeyVersionsResponse { + // The list of [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion]. + repeated CryptoKeyVersion crypto_key_versions = 1; + + // A token to retrieve next page of results. Pass this value in + // [ListCryptoKeyVersionsRequest.page_token][google.cloud.kms.v1.ListCryptoKeyVersionsRequest.page_token] + // to retrieve the next page of results. + string next_page_token = 2; + + // The total number of + // [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion] that matched the + // query. + int32 total_size = 3; +} + +// Response message for +// [KeyManagementService.ListImportJobs][google.cloud.kms.v1.KeyManagementService.ListImportJobs]. +message ListImportJobsResponse { + // The list of [ImportJobs][google.cloud.kms.v1.ImportJob]. + repeated ImportJob import_jobs = 1; + + // A token to retrieve next page of results. Pass this value in + // [ListImportJobsRequest.page_token][google.cloud.kms.v1.ListImportJobsRequest.page_token] + // to retrieve the next page of results. + string next_page_token = 2; + + // The total number of [ImportJobs][google.cloud.kms.v1.ImportJob] that + // matched the query. + int32 total_size = 3; +} + +// Request message for +// [KeyManagementService.GetKeyRing][google.cloud.kms.v1.KeyManagementService.GetKeyRing]. +message GetKeyRingRequest { + // Required. The [name][google.cloud.kms.v1.KeyRing.name] of the + // [KeyRing][google.cloud.kms.v1.KeyRing] to get. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/KeyRing" + } + ]; +} + +// Request message for +// [KeyManagementService.GetCryptoKey][google.cloud.kms.v1.KeyManagementService.GetCryptoKey]. +message GetCryptoKeyRequest { + // Required. The [name][google.cloud.kms.v1.CryptoKey.name] of the + // [CryptoKey][google.cloud.kms.v1.CryptoKey] to get. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKey" + } + ]; +} + +// Request message for +// [KeyManagementService.GetCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.GetCryptoKeyVersion]. +message GetCryptoKeyVersionRequest { + // Required. The [name][google.cloud.kms.v1.CryptoKeyVersion.name] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to get. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKeyVersion" + } + ]; +} + +// Request message for +// [KeyManagementService.GetPublicKey][google.cloud.kms.v1.KeyManagementService.GetPublicKey]. +message GetPublicKeyRequest { + // Required. The [name][google.cloud.kms.v1.CryptoKeyVersion.name] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] public key to get. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKeyVersion" + } + ]; +} + +// Request message for +// [KeyManagementService.GetImportJob][google.cloud.kms.v1.KeyManagementService.GetImportJob]. +message GetImportJobRequest { + // Required. The [name][google.cloud.kms.v1.ImportJob.name] of the + // [ImportJob][google.cloud.kms.v1.ImportJob] to get. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/ImportJob" + } + ]; +} + +// Request message for +// [KeyManagementService.CreateKeyRing][google.cloud.kms.v1.KeyManagementService.CreateKeyRing]. +message CreateKeyRingRequest { + // Required. The resource name of the location associated with the + // [KeyRings][google.cloud.kms.v1.KeyRing], in the format + // `projects/*/locations/*`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "locations.googleapis.com/Location" + } + ]; + + // Required. It must be unique within a location and match the regular + // expression `[a-zA-Z0-9_-]{1,63}` + string key_ring_id = 2 [(google.api.field_behavior) = REQUIRED]; + + // Required. A [KeyRing][google.cloud.kms.v1.KeyRing] with initial field + // values. + KeyRing key_ring = 3 [(google.api.field_behavior) = REQUIRED]; +} + +// Request message for +// [KeyManagementService.CreateCryptoKey][google.cloud.kms.v1.KeyManagementService.CreateCryptoKey]. +message CreateCryptoKeyRequest { + // Required. The [name][google.cloud.kms.v1.KeyRing.name] of the KeyRing + // associated with the [CryptoKeys][google.cloud.kms.v1.CryptoKey]. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/KeyRing" + } + ]; + + // Required. It must be unique within a KeyRing and match the regular + // expression `[a-zA-Z0-9_-]{1,63}` + string crypto_key_id = 2 [(google.api.field_behavior) = REQUIRED]; + + // Required. A [CryptoKey][google.cloud.kms.v1.CryptoKey] with initial field + // values. + CryptoKey crypto_key = 3 [(google.api.field_behavior) = REQUIRED]; + + // If set to true, the request will create a + // [CryptoKey][google.cloud.kms.v1.CryptoKey] without any + // [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion]. You must + // manually call + // [CreateCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.CreateCryptoKeyVersion] + // or + // [ImportCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.ImportCryptoKeyVersion] + // before you can use this [CryptoKey][google.cloud.kms.v1.CryptoKey]. + bool skip_initial_version_creation = 5; +} + +// Request message for +// [KeyManagementService.CreateCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.CreateCryptoKeyVersion]. +message CreateCryptoKeyVersionRequest { + // Required. The [name][google.cloud.kms.v1.CryptoKey.name] of the + // [CryptoKey][google.cloud.kms.v1.CryptoKey] associated with the + // [CryptoKeyVersions][google.cloud.kms.v1.CryptoKeyVersion]. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKey" + } + ]; + + // Required. A [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] with + // initial field values. + CryptoKeyVersion crypto_key_version = 2 + [(google.api.field_behavior) = REQUIRED]; +} + +// Request message for +// [KeyManagementService.ImportCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.ImportCryptoKeyVersion]. +message ImportCryptoKeyVersionRequest { + // Required. The [name][google.cloud.kms.v1.CryptoKey.name] of the + // [CryptoKey][google.cloud.kms.v1.CryptoKey] to be imported into. + // + // The create permission is only required on this key when creating a new + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKey" + } + ]; + + // Optional. The optional [name][google.cloud.kms.v1.CryptoKeyVersion.name] of + // an existing [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to + // target for an import operation. If this field is not present, a new + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] containing the + // supplied key material is created. + // + // If this field is present, the supplied key material is imported into + // the existing [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]. To + // import into an existing + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion], the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] must be a child of + // [ImportCryptoKeyVersionRequest.parent][google.cloud.kms.v1.ImportCryptoKeyVersionRequest.parent], + // have been previously created via [ImportCryptoKeyVersion][], and be in + // [DESTROYED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.DESTROYED] + // or + // [IMPORT_FAILED][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionState.IMPORT_FAILED] + // state. The key material and algorithm must match the previous + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] exactly if the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] has ever contained + // key material. + string crypto_key_version = 6 [ + (google.api.field_behavior) = OPTIONAL, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKeyVersion" + } + ]; + + // Required. The + // [algorithm][google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionAlgorithm] + // of the key being imported. This does not need to match the + // [version_template][google.cloud.kms.v1.CryptoKey.version_template] of the + // [CryptoKey][google.cloud.kms.v1.CryptoKey] this version imports into. + CryptoKeyVersion.CryptoKeyVersionAlgorithm algorithm = 2 + [(google.api.field_behavior) = REQUIRED]; + + // Required. The [name][google.cloud.kms.v1.ImportJob.name] of the + // [ImportJob][google.cloud.kms.v1.ImportJob] that was used to wrap this key + // material. + string import_job = 4 [(google.api.field_behavior) = REQUIRED]; + + // Optional. The wrapped key material to import. + // + // Before wrapping, key material must be formatted. If importing symmetric key + // material, the expected key material format is plain bytes. If importing + // asymmetric key material, the expected key material format is PKCS#8-encoded + // DER (the PrivateKeyInfo structure from RFC 5208). + // + // When wrapping with import methods + // ([RSA_OAEP_3072_SHA1_AES_256][google.cloud.kms.v1.ImportJob.ImportMethod.RSA_OAEP_3072_SHA1_AES_256] + // or + // [RSA_OAEP_4096_SHA1_AES_256][google.cloud.kms.v1.ImportJob.ImportMethod.RSA_OAEP_4096_SHA1_AES_256] + // or + // [RSA_OAEP_3072_SHA256_AES_256][google.cloud.kms.v1.ImportJob.ImportMethod.RSA_OAEP_3072_SHA256_AES_256] + // or + // [RSA_OAEP_4096_SHA256_AES_256][google.cloud.kms.v1.ImportJob.ImportMethod.RSA_OAEP_4096_SHA256_AES_256]), + // + // this field must contain the concatenation of: + //
    + //
  1. An ephemeral AES-256 wrapping key wrapped with the + // [public_key][google.cloud.kms.v1.ImportJob.public_key] using + // RSAES-OAEP with SHA-1/SHA-256, MGF1 with SHA-1/SHA-256, and an empty + // label. + //
  2. + //
  3. The formatted key to be imported, wrapped with the ephemeral AES-256 + // key using AES-KWP (RFC 5649). + //
  4. + //
+ // + // This format is the same as the format produced by PKCS#11 mechanism + // CKM_RSA_AES_KEY_WRAP. + // + // When wrapping with import methods + // ([RSA_OAEP_3072_SHA256][google.cloud.kms.v1.ImportJob.ImportMethod.RSA_OAEP_3072_SHA256] + // or + // [RSA_OAEP_4096_SHA256][google.cloud.kms.v1.ImportJob.ImportMethod.RSA_OAEP_4096_SHA256]), + // + // this field must contain the formatted key to be imported, wrapped with the + // [public_key][google.cloud.kms.v1.ImportJob.public_key] using RSAES-OAEP + // with SHA-256, MGF1 with SHA-256, and an empty label. + bytes wrapped_key = 8 [(google.api.field_behavior) = OPTIONAL]; + + // This field is legacy. Use the field + // [wrapped_key][google.cloud.kms.v1.ImportCryptoKeyVersionRequest.wrapped_key] + // instead. + oneof wrapped_key_material { + // Optional. This field has the same meaning as + // [wrapped_key][google.cloud.kms.v1.ImportCryptoKeyVersionRequest.wrapped_key]. + // Prefer to use that field in new work. Either that field or this field + // (but not both) must be specified. + bytes rsa_aes_wrapped_key = 5 [(google.api.field_behavior) = OPTIONAL]; + } +} + +// Request message for +// [KeyManagementService.CreateImportJob][google.cloud.kms.v1.KeyManagementService.CreateImportJob]. +message CreateImportJobRequest { + // Required. The [name][google.cloud.kms.v1.KeyRing.name] of the + // [KeyRing][google.cloud.kms.v1.KeyRing] associated with the + // [ImportJobs][google.cloud.kms.v1.ImportJob]. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/KeyRing" + } + ]; + + // Required. It must be unique within a KeyRing and match the regular + // expression `[a-zA-Z0-9_-]{1,63}` + string import_job_id = 2 [(google.api.field_behavior) = REQUIRED]; + + // Required. An [ImportJob][google.cloud.kms.v1.ImportJob] with initial field + // values. + ImportJob import_job = 3 [(google.api.field_behavior) = REQUIRED]; +} + +// Request message for +// [KeyManagementService.UpdateCryptoKey][google.cloud.kms.v1.KeyManagementService.UpdateCryptoKey]. +message UpdateCryptoKeyRequest { + // Required. [CryptoKey][google.cloud.kms.v1.CryptoKey] with updated values. + CryptoKey crypto_key = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. List of fields to be updated in this request. + google.protobuf.FieldMask update_mask = 2 + [(google.api.field_behavior) = REQUIRED]; +} + +// Request message for +// [KeyManagementService.UpdateCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.UpdateCryptoKeyVersion]. +message UpdateCryptoKeyVersionRequest { + // Required. [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] with + // updated values. + CryptoKeyVersion crypto_key_version = 1 + [(google.api.field_behavior) = REQUIRED]; + + // Required. List of fields to be updated in this request. + google.protobuf.FieldMask update_mask = 2 + [(google.api.field_behavior) = REQUIRED]; +} + +// Request message for +// [KeyManagementService.UpdateCryptoKeyPrimaryVersion][google.cloud.kms.v1.KeyManagementService.UpdateCryptoKeyPrimaryVersion]. +message UpdateCryptoKeyPrimaryVersionRequest { + // Required. The resource name of the + // [CryptoKey][google.cloud.kms.v1.CryptoKey] to update. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKey" + } + ]; + + // Required. The id of the child + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to use as primary. + string crypto_key_version_id = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// Request message for +// [KeyManagementService.DestroyCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.DestroyCryptoKeyVersion]. +message DestroyCryptoKeyVersionRequest { + // Required. The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to destroy. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKeyVersion" + } + ]; +} + +// Request message for +// [KeyManagementService.RestoreCryptoKeyVersion][google.cloud.kms.v1.KeyManagementService.RestoreCryptoKeyVersion]. +message RestoreCryptoKeyVersionRequest { + // Required. The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to restore. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKeyVersion" + } + ]; +} + +// Request message for +// [KeyManagementService.Encrypt][google.cloud.kms.v1.KeyManagementService.Encrypt]. +message EncryptRequest { + // Required. The resource name of the + // [CryptoKey][google.cloud.kms.v1.CryptoKey] or + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to use for + // encryption. + // + // If a [CryptoKey][google.cloud.kms.v1.CryptoKey] is specified, the server + // will use its [primary version][google.cloud.kms.v1.CryptoKey.primary]. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "*" } + ]; + + // Required. The data to encrypt. Must be no larger than 64KiB. + // + // The maximum size depends on the key version's + // [protection_level][google.cloud.kms.v1.CryptoKeyVersionTemplate.protection_level]. + // For [SOFTWARE][google.cloud.kms.v1.ProtectionLevel.SOFTWARE], + // [EXTERNAL][google.cloud.kms.v1.ProtectionLevel.EXTERNAL], and + // [EXTERNAL_VPC][google.cloud.kms.v1.ProtectionLevel.EXTERNAL_VPC] keys, the + // plaintext must be no larger than 64KiB. For + // [HSM][google.cloud.kms.v1.ProtectionLevel.HSM] keys, the combined length of + // the plaintext and additional_authenticated_data fields must be no larger + // than 8KiB. + bytes plaintext = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Optional data that, if specified, must also be provided during + // decryption through + // [DecryptRequest.additional_authenticated_data][google.cloud.kms.v1.DecryptRequest.additional_authenticated_data]. + // + // The maximum size depends on the key version's + // [protection_level][google.cloud.kms.v1.CryptoKeyVersionTemplate.protection_level]. + // For [SOFTWARE][google.cloud.kms.v1.ProtectionLevel.SOFTWARE], + // [EXTERNAL][google.cloud.kms.v1.ProtectionLevel.EXTERNAL], and + // [EXTERNAL_VPC][google.cloud.kms.v1.ProtectionLevel.EXTERNAL_VPC] keys the + // AAD must be no larger than 64KiB. For + // [HSM][google.cloud.kms.v1.ProtectionLevel.HSM] keys, the combined length of + // the plaintext and additional_authenticated_data fields must be no larger + // than 8KiB. + bytes additional_authenticated_data = 3 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional CRC32C checksum of the + // [EncryptRequest.plaintext][google.cloud.kms.v1.EncryptRequest.plaintext]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received + // [EncryptRequest.plaintext][google.cloud.kms.v1.EncryptRequest.plaintext] + // using this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C([EncryptRequest.plaintext][google.cloud.kms.v1.EncryptRequest.plaintext]) + // is equal to + // [EncryptRequest.plaintext_crc32c][google.cloud.kms.v1.EncryptRequest.plaintext_crc32c], + // and if so, perform a limited number of retries. A persistent mismatch may + // indicate an issue in your computation of the CRC32C checksum. Note: This + // field is defined as int64 for reasons of compatibility across different + // languages. However, it is a non-negative integer, which will never exceed + // 2^32-1, and can be safely downconverted to uint32 in languages that support + // this type. + google.protobuf.Int64Value plaintext_crc32c = 7 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional CRC32C checksum of the + // [EncryptRequest.additional_authenticated_data][google.cloud.kms.v1.EncryptRequest.additional_authenticated_data]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received + // [EncryptRequest.additional_authenticated_data][google.cloud.kms.v1.EncryptRequest.additional_authenticated_data] + // using this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C([EncryptRequest.additional_authenticated_data][google.cloud.kms.v1.EncryptRequest.additional_authenticated_data]) + // is equal to + // [EncryptRequest.additional_authenticated_data_crc32c][google.cloud.kms.v1.EncryptRequest.additional_authenticated_data_crc32c], + // and if so, perform a limited number of retries. A persistent mismatch may + // indicate an issue in your computation of the CRC32C checksum. Note: This + // field is defined as int64 for reasons of compatibility across different + // languages. However, it is a non-negative integer, which will never exceed + // 2^32-1, and can be safely downconverted to uint32 in languages that support + // this type. + google.protobuf.Int64Value additional_authenticated_data_crc32c = 8 + [(google.api.field_behavior) = OPTIONAL]; +} + +// Request message for +// [KeyManagementService.Decrypt][google.cloud.kms.v1.KeyManagementService.Decrypt]. +message DecryptRequest { + // Required. The resource name of the + // [CryptoKey][google.cloud.kms.v1.CryptoKey] to use for decryption. The + // server will choose the appropriate version. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKey" + } + ]; + + // Required. The encrypted data originally returned in + // [EncryptResponse.ciphertext][google.cloud.kms.v1.EncryptResponse.ciphertext]. + bytes ciphertext = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Optional data that must match the data originally supplied in + // [EncryptRequest.additional_authenticated_data][google.cloud.kms.v1.EncryptRequest.additional_authenticated_data]. + bytes additional_authenticated_data = 3 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional CRC32C checksum of the + // [DecryptRequest.ciphertext][google.cloud.kms.v1.DecryptRequest.ciphertext]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received + // [DecryptRequest.ciphertext][google.cloud.kms.v1.DecryptRequest.ciphertext] + // using this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C([DecryptRequest.ciphertext][google.cloud.kms.v1.DecryptRequest.ciphertext]) + // is equal to + // [DecryptRequest.ciphertext_crc32c][google.cloud.kms.v1.DecryptRequest.ciphertext_crc32c], + // and if so, perform a limited number of retries. A persistent mismatch may + // indicate an issue in your computation of the CRC32C checksum. Note: This + // field is defined as int64 for reasons of compatibility across different + // languages. However, it is a non-negative integer, which will never exceed + // 2^32-1, and can be safely downconverted to uint32 in languages that support + // this type. + google.protobuf.Int64Value ciphertext_crc32c = 5 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional CRC32C checksum of the + // [DecryptRequest.additional_authenticated_data][google.cloud.kms.v1.DecryptRequest.additional_authenticated_data]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received + // [DecryptRequest.additional_authenticated_data][google.cloud.kms.v1.DecryptRequest.additional_authenticated_data] + // using this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C([DecryptRequest.additional_authenticated_data][google.cloud.kms.v1.DecryptRequest.additional_authenticated_data]) + // is equal to + // [DecryptRequest.additional_authenticated_data_crc32c][google.cloud.kms.v1.DecryptRequest.additional_authenticated_data_crc32c], + // and if so, perform a limited number of retries. A persistent mismatch may + // indicate an issue in your computation of the CRC32C checksum. Note: This + // field is defined as int64 for reasons of compatibility across different + // languages. However, it is a non-negative integer, which will never exceed + // 2^32-1, and can be safely downconverted to uint32 in languages that support + // this type. + google.protobuf.Int64Value additional_authenticated_data_crc32c = 6 + [(google.api.field_behavior) = OPTIONAL]; +} + +// Request message for +// [KeyManagementService.RawEncrypt][google.cloud.kms.v1.KeyManagementService.RawEncrypt]. +message RawEncryptRequest { + // Required. The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to use for + // encryption. + string name = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. The data to encrypt. Must be no larger than 64KiB. + // + // The maximum size depends on the key version's + // [protection_level][google.cloud.kms.v1.CryptoKeyVersionTemplate.protection_level]. + // For [SOFTWARE][google.cloud.kms.v1.ProtectionLevel.SOFTWARE] keys, the + // plaintext must be no larger than 64KiB. For + // [HSM][google.cloud.kms.v1.ProtectionLevel.HSM] keys, the combined length of + // the plaintext and additional_authenticated_data fields must be no larger + // than 8KiB. + bytes plaintext = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Optional data that, if specified, must also be provided during + // decryption through + // [RawDecryptRequest.additional_authenticated_data][google.cloud.kms.v1.RawDecryptRequest.additional_authenticated_data]. + // + // This field may only be used in conjunction with an + // [algorithm][google.cloud.kms.v1.CryptoKeyVersion.algorithm] that accepts + // additional authenticated data (for example, AES-GCM). + // + // The maximum size depends on the key version's + // [protection_level][google.cloud.kms.v1.CryptoKeyVersionTemplate.protection_level]. + // For [SOFTWARE][google.cloud.kms.v1.ProtectionLevel.SOFTWARE] keys, the + // plaintext must be no larger than 64KiB. For + // [HSM][google.cloud.kms.v1.ProtectionLevel.HSM] keys, the combined length of + // the plaintext and additional_authenticated_data fields must be no larger + // than 8KiB. + bytes additional_authenticated_data = 3 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional CRC32C checksum of the + // [RawEncryptRequest.plaintext][google.cloud.kms.v1.RawEncryptRequest.plaintext]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received plaintext using this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that CRC32C(plaintext) is equal + // to plaintext_crc32c, and if so, perform a limited number of retries. A + // persistent mismatch may indicate an issue in your computation of the CRC32C + // checksum. Note: This field is defined as int64 for reasons of compatibility + // across different languages. However, it is a non-negative integer, which + // will never exceed 2^32-1, and can be safely downconverted to uint32 in + // languages that support this type. + google.protobuf.Int64Value plaintext_crc32c = 4 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional CRC32C checksum of the + // [RawEncryptRequest.additional_authenticated_data][google.cloud.kms.v1.RawEncryptRequest.additional_authenticated_data]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received additional_authenticated_data using + // this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C(additional_authenticated_data) is equal to + // additional_authenticated_data_crc32c, and if so, perform + // a limited number of retries. A persistent mismatch may indicate an issue in + // your computation of the CRC32C checksum. + // Note: This field is defined as int64 for reasons of compatibility across + // different languages. However, it is a non-negative integer, which will + // never exceed 2^32-1, and can be safely downconverted to uint32 in languages + // that support this type. + google.protobuf.Int64Value additional_authenticated_data_crc32c = 5 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A customer-supplied initialization vector that will be used for + // encryption. If it is not provided for AES-CBC and AES-CTR, one will be + // generated. It will be returned in + // [RawEncryptResponse.initialization_vector][google.cloud.kms.v1.RawEncryptResponse.initialization_vector]. + bytes initialization_vector = 6 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional CRC32C checksum of the + // [RawEncryptRequest.initialization_vector][google.cloud.kms.v1.RawEncryptRequest.initialization_vector]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received initialization_vector using this + // checksum. [KeyManagementService][google.cloud.kms.v1.KeyManagementService] + // will report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C(initialization_vector) is equal to + // initialization_vector_crc32c, and if so, perform + // a limited number of retries. A persistent mismatch may indicate an issue in + // your computation of the CRC32C checksum. + // Note: This field is defined as int64 for reasons of compatibility across + // different languages. However, it is a non-negative integer, which will + // never exceed 2^32-1, and can be safely downconverted to uint32 in languages + // that support this type. + google.protobuf.Int64Value initialization_vector_crc32c = 7 + [(google.api.field_behavior) = OPTIONAL]; +} + +// Request message for +// [KeyManagementService.RawDecrypt][google.cloud.kms.v1.KeyManagementService.RawDecrypt]. +message RawDecryptRequest { + // Required. The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to use for + // decryption. + string name = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. The encrypted data originally returned in + // [RawEncryptResponse.ciphertext][google.cloud.kms.v1.RawEncryptResponse.ciphertext]. + bytes ciphertext = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Optional data that must match the data originally supplied in + // [RawEncryptRequest.additional_authenticated_data][google.cloud.kms.v1.RawEncryptRequest.additional_authenticated_data]. + bytes additional_authenticated_data = 3 + [(google.api.field_behavior) = OPTIONAL]; + + // Required. The initialization vector (IV) used during encryption, which must + // match the data originally provided in + // [RawEncryptResponse.initialization_vector][google.cloud.kms.v1.RawEncryptResponse.initialization_vector]. + bytes initialization_vector = 4 [(google.api.field_behavior) = REQUIRED]; + + // The length of the authentication tag that is appended to the end of + // the ciphertext. If unspecified (0), the default value for the key's + // algorithm will be used (for AES-GCM, the default value is 16). + int32 tag_length = 5; + + // Optional. An optional CRC32C checksum of the + // [RawDecryptRequest.ciphertext][google.cloud.kms.v1.RawDecryptRequest.ciphertext]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received ciphertext using this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that CRC32C(ciphertext) is equal + // to ciphertext_crc32c, and if so, perform a limited number of retries. A + // persistent mismatch may indicate an issue in your computation of the CRC32C + // checksum. Note: This field is defined as int64 for reasons of compatibility + // across different languages. However, it is a non-negative integer, which + // will never exceed 2^32-1, and can be safely downconverted to uint32 in + // languages that support this type. + google.protobuf.Int64Value ciphertext_crc32c = 6 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional CRC32C checksum of the + // [RawDecryptRequest.additional_authenticated_data][google.cloud.kms.v1.RawDecryptRequest.additional_authenticated_data]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received additional_authenticated_data using + // this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C(additional_authenticated_data) is equal to + // additional_authenticated_data_crc32c, and if so, perform + // a limited number of retries. A persistent mismatch may indicate an issue in + // your computation of the CRC32C checksum. + // Note: This field is defined as int64 for reasons of compatibility across + // different languages. However, it is a non-negative integer, which will + // never exceed 2^32-1, and can be safely downconverted to uint32 in languages + // that support this type. + google.protobuf.Int64Value additional_authenticated_data_crc32c = 7 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional CRC32C checksum of the + // [RawDecryptRequest.initialization_vector][google.cloud.kms.v1.RawDecryptRequest.initialization_vector]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received initialization_vector using this + // checksum. [KeyManagementService][google.cloud.kms.v1.KeyManagementService] + // will report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C(initialization_vector) is equal to initialization_vector_crc32c, and + // if so, perform a limited number of retries. A persistent mismatch may + // indicate an issue in your computation of the CRC32C checksum. + // Note: This field is defined as int64 for reasons of compatibility across + // different languages. However, it is a non-negative integer, which will + // never exceed 2^32-1, and can be safely downconverted to uint32 in languages + // that support this type. + google.protobuf.Int64Value initialization_vector_crc32c = 8 + [(google.api.field_behavior) = OPTIONAL]; +} + +// Request message for +// [KeyManagementService.AsymmetricSign][google.cloud.kms.v1.KeyManagementService.AsymmetricSign]. +message AsymmetricSignRequest { + // Required. The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to use for + // signing. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKeyVersion" + } + ]; + + // Optional. The digest of the data to sign. The digest must be produced with + // the same digest algorithm as specified by the key version's + // [algorithm][google.cloud.kms.v1.CryptoKeyVersion.algorithm]. + // + // This field may not be supplied if + // [AsymmetricSignRequest.data][google.cloud.kms.v1.AsymmetricSignRequest.data] + // is supplied. + Digest digest = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional CRC32C checksum of the + // [AsymmetricSignRequest.digest][google.cloud.kms.v1.AsymmetricSignRequest.digest]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received + // [AsymmetricSignRequest.digest][google.cloud.kms.v1.AsymmetricSignRequest.digest] + // using this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C([AsymmetricSignRequest.digest][google.cloud.kms.v1.AsymmetricSignRequest.digest]) + // is equal to + // [AsymmetricSignRequest.digest_crc32c][google.cloud.kms.v1.AsymmetricSignRequest.digest_crc32c], + // and if so, perform a limited number of retries. A persistent mismatch may + // indicate an issue in your computation of the CRC32C checksum. Note: This + // field is defined as int64 for reasons of compatibility across different + // languages. However, it is a non-negative integer, which will never exceed + // 2^32-1, and can be safely downconverted to uint32 in languages that support + // this type. + google.protobuf.Int64Value digest_crc32c = 4 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The data to sign. + // It can't be supplied if + // [AsymmetricSignRequest.digest][google.cloud.kms.v1.AsymmetricSignRequest.digest] + // is supplied. + bytes data = 6 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional CRC32C checksum of the + // [AsymmetricSignRequest.data][google.cloud.kms.v1.AsymmetricSignRequest.data]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received + // [AsymmetricSignRequest.data][google.cloud.kms.v1.AsymmetricSignRequest.data] + // using this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C([AsymmetricSignRequest.data][google.cloud.kms.v1.AsymmetricSignRequest.data]) + // is equal to + // [AsymmetricSignRequest.data_crc32c][google.cloud.kms.v1.AsymmetricSignRequest.data_crc32c], + // and if so, perform a limited number of retries. A persistent mismatch may + // indicate an issue in your computation of the CRC32C checksum. Note: This + // field is defined as int64 for reasons of compatibility across different + // languages. However, it is a non-negative integer, which will never exceed + // 2^32-1, and can be safely downconverted to uint32 in languages that support + // this type. + google.protobuf.Int64Value data_crc32c = 7 + [(google.api.field_behavior) = OPTIONAL]; +} + +// Request message for +// [KeyManagementService.AsymmetricDecrypt][google.cloud.kms.v1.KeyManagementService.AsymmetricDecrypt]. +message AsymmetricDecryptRequest { + // Required. The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to use for + // decryption. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKeyVersion" + } + ]; + + // Required. The data encrypted with the named + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion]'s public key using + // OAEP. + bytes ciphertext = 3 [(google.api.field_behavior) = REQUIRED]; + + // Optional. An optional CRC32C checksum of the + // [AsymmetricDecryptRequest.ciphertext][google.cloud.kms.v1.AsymmetricDecryptRequest.ciphertext]. + // If specified, + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // verify the integrity of the received + // [AsymmetricDecryptRequest.ciphertext][google.cloud.kms.v1.AsymmetricDecryptRequest.ciphertext] + // using this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C([AsymmetricDecryptRequest.ciphertext][google.cloud.kms.v1.AsymmetricDecryptRequest.ciphertext]) + // is equal to + // [AsymmetricDecryptRequest.ciphertext_crc32c][google.cloud.kms.v1.AsymmetricDecryptRequest.ciphertext_crc32c], + // and if so, perform a limited number of retries. A persistent mismatch may + // indicate an issue in your computation of the CRC32C checksum. Note: This + // field is defined as int64 for reasons of compatibility across different + // languages. However, it is a non-negative integer, which will never exceed + // 2^32-1, and can be safely downconverted to uint32 in languages that support + // this type. + google.protobuf.Int64Value ciphertext_crc32c = 4 + [(google.api.field_behavior) = OPTIONAL]; +} + +// Request message for +// [KeyManagementService.MacSign][google.cloud.kms.v1.KeyManagementService.MacSign]. +message MacSignRequest { + // Required. The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to use for + // signing. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKeyVersion" + } + ]; + + // Required. The data to sign. The MAC tag is computed over this data field + // based on the specific algorithm. + bytes data = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. An optional CRC32C checksum of the + // [MacSignRequest.data][google.cloud.kms.v1.MacSignRequest.data]. If + // specified, [KeyManagementService][google.cloud.kms.v1.KeyManagementService] + // will verify the integrity of the received + // [MacSignRequest.data][google.cloud.kms.v1.MacSignRequest.data] using this + // checksum. [KeyManagementService][google.cloud.kms.v1.KeyManagementService] + // will report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C([MacSignRequest.data][google.cloud.kms.v1.MacSignRequest.data]) is + // equal to + // [MacSignRequest.data_crc32c][google.cloud.kms.v1.MacSignRequest.data_crc32c], + // and if so, perform a limited number of retries. A persistent mismatch may + // indicate an issue in your computation of the CRC32C checksum. Note: This + // field is defined as int64 for reasons of compatibility across different + // languages. However, it is a non-negative integer, which will never exceed + // 2^32-1, and can be safely downconverted to uint32 in languages that support + // this type. + google.protobuf.Int64Value data_crc32c = 3 + [(google.api.field_behavior) = OPTIONAL]; +} + +// Request message for +// [KeyManagementService.MacVerify][google.cloud.kms.v1.KeyManagementService.MacVerify]. +message MacVerifyRequest { + // Required. The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] to use for + // verification. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudkms.googleapis.com/CryptoKeyVersion" + } + ]; + + // Required. The data used previously as a + // [MacSignRequest.data][google.cloud.kms.v1.MacSignRequest.data] to generate + // the MAC tag. + bytes data = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. An optional CRC32C checksum of the + // [MacVerifyRequest.data][google.cloud.kms.v1.MacVerifyRequest.data]. If + // specified, [KeyManagementService][google.cloud.kms.v1.KeyManagementService] + // will verify the integrity of the received + // [MacVerifyRequest.data][google.cloud.kms.v1.MacVerifyRequest.data] using + // this checksum. + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] will + // report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C([MacVerifyRequest.data][google.cloud.kms.v1.MacVerifyRequest.data]) + // is equal to + // [MacVerifyRequest.data_crc32c][google.cloud.kms.v1.MacVerifyRequest.data_crc32c], + // and if so, perform a limited number of retries. A persistent mismatch may + // indicate an issue in your computation of the CRC32C checksum. Note: This + // field is defined as int64 for reasons of compatibility across different + // languages. However, it is a non-negative integer, which will never exceed + // 2^32-1, and can be safely downconverted to uint32 in languages that support + // this type. + google.protobuf.Int64Value data_crc32c = 3 + [(google.api.field_behavior) = OPTIONAL]; + + // Required. The signature to verify. + bytes mac = 4 [(google.api.field_behavior) = REQUIRED]; + + // Optional. An optional CRC32C checksum of the + // [MacVerifyRequest.mac][google.cloud.kms.v1.MacVerifyRequest.mac]. If + // specified, [KeyManagementService][google.cloud.kms.v1.KeyManagementService] + // will verify the integrity of the received + // [MacVerifyRequest.mac][google.cloud.kms.v1.MacVerifyRequest.mac] using this + // checksum. [KeyManagementService][google.cloud.kms.v1.KeyManagementService] + // will report an error if the checksum verification fails. If you receive a + // checksum error, your client should verify that + // CRC32C([MacVerifyRequest.tag][]) is equal to + // [MacVerifyRequest.mac_crc32c][google.cloud.kms.v1.MacVerifyRequest.mac_crc32c], + // and if so, perform a limited number of retries. A persistent mismatch may + // indicate an issue in your computation of the CRC32C checksum. Note: This + // field is defined as int64 for reasons of compatibility across different + // languages. However, it is a non-negative integer, which will never exceed + // 2^32-1, and can be safely downconverted to uint32 in languages that support + // this type. + google.protobuf.Int64Value mac_crc32c = 5 + [(google.api.field_behavior) = OPTIONAL]; +} + +// Request message for +// [KeyManagementService.GenerateRandomBytes][google.cloud.kms.v1.KeyManagementService.GenerateRandomBytes]. +message GenerateRandomBytesRequest { + // The project-specific location in which to generate random bytes. + // For example, "projects/my-project/locations/us-central1". + string location = 1; + + // The length in bytes of the amount of randomness to retrieve. Minimum 8 + // bytes, maximum 1024 bytes. + int32 length_bytes = 2; + + // The [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] to use when + // generating the random data. Currently, only + // [HSM][google.cloud.kms.v1.ProtectionLevel.HSM] protection level is + // supported. + ProtectionLevel protection_level = 3; +} + +// Response message for +// [KeyManagementService.Encrypt][google.cloud.kms.v1.KeyManagementService.Encrypt]. +message EncryptResponse { + // The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used in + // encryption. Check this field to verify that the intended resource was used + // for encryption. + string name = 1; + + // The encrypted data. + bytes ciphertext = 2; + + // Integrity verification field. A CRC32C checksum of the returned + // [EncryptResponse.ciphertext][google.cloud.kms.v1.EncryptResponse.ciphertext]. + // An integrity check of + // [EncryptResponse.ciphertext][google.cloud.kms.v1.EncryptResponse.ciphertext] + // can be performed by computing the CRC32C checksum of + // [EncryptResponse.ciphertext][google.cloud.kms.v1.EncryptResponse.ciphertext] + // and comparing your results to this field. Discard the response in case of + // non-matching checksum values, and perform a limited number of retries. A + // persistent mismatch may indicate an issue in your computation of the CRC32C + // checksum. Note: This field is defined as int64 for reasons of compatibility + // across different languages. However, it is a non-negative integer, which + // will never exceed 2^32-1, and can be safely downconverted to uint32 in + // languages that support this type. + google.protobuf.Int64Value ciphertext_crc32c = 4; + + // Integrity verification field. A flag indicating whether + // [EncryptRequest.plaintext_crc32c][google.cloud.kms.v1.EncryptRequest.plaintext_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of the + // [plaintext][google.cloud.kms.v1.EncryptRequest.plaintext]. A false value of + // this field indicates either that + // [EncryptRequest.plaintext_crc32c][google.cloud.kms.v1.EncryptRequest.plaintext_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [EncryptRequest.plaintext_crc32c][google.cloud.kms.v1.EncryptRequest.plaintext_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_plaintext_crc32c = 5; + + // Integrity verification field. A flag indicating whether + // [EncryptRequest.additional_authenticated_data_crc32c][google.cloud.kms.v1.EncryptRequest.additional_authenticated_data_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of the + // [AAD][google.cloud.kms.v1.EncryptRequest.additional_authenticated_data]. A + // false value of this field indicates either that + // [EncryptRequest.additional_authenticated_data_crc32c][google.cloud.kms.v1.EncryptRequest.additional_authenticated_data_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [EncryptRequest.additional_authenticated_data_crc32c][google.cloud.kms.v1.EncryptRequest.additional_authenticated_data_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_additional_authenticated_data_crc32c = 6; + + // The [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used in + // encryption. + ProtectionLevel protection_level = 7; +} + +// Response message for +// [KeyManagementService.Decrypt][google.cloud.kms.v1.KeyManagementService.Decrypt]. +message DecryptResponse { + // The decrypted data originally supplied in + // [EncryptRequest.plaintext][google.cloud.kms.v1.EncryptRequest.plaintext]. + bytes plaintext = 1; + + // Integrity verification field. A CRC32C checksum of the returned + // [DecryptResponse.plaintext][google.cloud.kms.v1.DecryptResponse.plaintext]. + // An integrity check of + // [DecryptResponse.plaintext][google.cloud.kms.v1.DecryptResponse.plaintext] + // can be performed by computing the CRC32C checksum of + // [DecryptResponse.plaintext][google.cloud.kms.v1.DecryptResponse.plaintext] + // and comparing your results to this field. Discard the response in case of + // non-matching checksum values, and perform a limited number of retries. A + // persistent mismatch may indicate an issue in your computation of the CRC32C + // checksum. Note: receiving this response message indicates that + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] is able to + // successfully decrypt the + // [ciphertext][google.cloud.kms.v1.DecryptRequest.ciphertext]. Note: This + // field is defined as int64 for reasons of compatibility across different + // languages. However, it is a non-negative integer, which will never exceed + // 2^32-1, and can be safely downconverted to uint32 in languages that support + // this type. + google.protobuf.Int64Value plaintext_crc32c = 2; + + // Whether the Decryption was performed using the primary key version. + bool used_primary = 3; + + // The [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used in + // decryption. + ProtectionLevel protection_level = 4; +} + +// Response message for +// [KeyManagementService.RawEncrypt][google.cloud.kms.v1.KeyManagementService.RawEncrypt]. +message RawEncryptResponse { + // The encrypted data. In the case of AES-GCM, the authentication tag + // is the [tag_length][google.cloud.kms.v1.RawEncryptResponse.tag_length] + // bytes at the end of this field. + bytes ciphertext = 1; + + // The initialization vector (IV) generated by the service during + // encryption. This value must be stored and provided in + // [RawDecryptRequest.initialization_vector][google.cloud.kms.v1.RawDecryptRequest.initialization_vector] + // at decryption time. + bytes initialization_vector = 2; + + // The length of the authentication tag that is appended to + // the end of the ciphertext. + int32 tag_length = 3; + + // Integrity verification field. A CRC32C checksum of the returned + // [RawEncryptResponse.ciphertext][google.cloud.kms.v1.RawEncryptResponse.ciphertext]. + // An integrity check of ciphertext can be performed by computing the CRC32C + // checksum of ciphertext and comparing your results to this field. Discard + // the response in case of non-matching checksum values, and perform a limited + // number of retries. A persistent mismatch may indicate an issue in your + // computation of the CRC32C checksum. Note: This field is defined as int64 + // for reasons of compatibility across different languages. However, it is a + // non-negative integer, which will never exceed 2^32-1, and can be safely + // downconverted to uint32 in languages that support this type. + google.protobuf.Int64Value ciphertext_crc32c = 4; + + // Integrity verification field. A CRC32C checksum of the returned + // [RawEncryptResponse.initialization_vector][google.cloud.kms.v1.RawEncryptResponse.initialization_vector]. + // An integrity check of initialization_vector can be performed by computing + // the CRC32C checksum of initialization_vector and comparing your results to + // this field. Discard the response in case of non-matching checksum values, + // and perform a limited number of retries. A persistent mismatch may indicate + // an issue in your computation of the CRC32C checksum. Note: This field is + // defined as int64 for reasons of compatibility across different languages. + // However, it is a non-negative integer, which will never exceed 2^32-1, and + // can be safely downconverted to uint32 in languages that support this type. + google.protobuf.Int64Value initialization_vector_crc32c = 5; + + // Integrity verification field. A flag indicating whether + // [RawEncryptRequest.plaintext_crc32c][google.cloud.kms.v1.RawEncryptRequest.plaintext_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of the plaintext. A false value of this + // field indicates either that + // [RawEncryptRequest.plaintext_crc32c][google.cloud.kms.v1.RawEncryptRequest.plaintext_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [RawEncryptRequest.plaintext_crc32c][google.cloud.kms.v1.RawEncryptRequest.plaintext_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_plaintext_crc32c = 6; + + // Integrity verification field. A flag indicating whether + // [RawEncryptRequest.additional_authenticated_data_crc32c][google.cloud.kms.v1.RawEncryptRequest.additional_authenticated_data_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of additional_authenticated_data. A false + // value of this field indicates either that // + // [RawEncryptRequest.additional_authenticated_data_crc32c][google.cloud.kms.v1.RawEncryptRequest.additional_authenticated_data_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [RawEncryptRequest.additional_authenticated_data_crc32c][google.cloud.kms.v1.RawEncryptRequest.additional_authenticated_data_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_additional_authenticated_data_crc32c = 7; + + // Integrity verification field. A flag indicating whether + // [RawEncryptRequest.initialization_vector_crc32c][google.cloud.kms.v1.RawEncryptRequest.initialization_vector_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of initialization_vector. A false value of + // this field indicates either that + // [RawEncryptRequest.initialization_vector_crc32c][google.cloud.kms.v1.RawEncryptRequest.initialization_vector_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [RawEncryptRequest.initialization_vector_crc32c][google.cloud.kms.v1.RawEncryptRequest.initialization_vector_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_initialization_vector_crc32c = 10; + + // The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used in + // encryption. Check this field to verify that the intended resource was used + // for encryption. + string name = 8; + + // The [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used in + // encryption. + ProtectionLevel protection_level = 9; +} + +// Response message for +// [KeyManagementService.RawDecrypt][google.cloud.kms.v1.KeyManagementService.RawDecrypt]. +message RawDecryptResponse { + // The decrypted data. + bytes plaintext = 1; + + // Integrity verification field. A CRC32C checksum of the returned + // [RawDecryptResponse.plaintext][google.cloud.kms.v1.RawDecryptResponse.plaintext]. + // An integrity check of plaintext can be performed by computing the CRC32C + // checksum of plaintext and comparing your results to this field. Discard the + // response in case of non-matching checksum values, and perform a limited + // number of retries. A persistent mismatch may indicate an issue in your + // computation of the CRC32C checksum. Note: receiving this response message + // indicates that + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] is able to + // successfully decrypt the + // [ciphertext][google.cloud.kms.v1.RawDecryptRequest.ciphertext]. + // Note: This field is defined as int64 for reasons of compatibility across + // different languages. However, it is a non-negative integer, which will + // never exceed 2^32-1, and can be safely downconverted to uint32 in languages + // that support this type. + google.protobuf.Int64Value plaintext_crc32c = 2; + + // The [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used in + // decryption. + ProtectionLevel protection_level = 3; + + // Integrity verification field. A flag indicating whether + // [RawDecryptRequest.ciphertext_crc32c][google.cloud.kms.v1.RawDecryptRequest.ciphertext_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of the ciphertext. A false value of this + // field indicates either that + // [RawDecryptRequest.ciphertext_crc32c][google.cloud.kms.v1.RawDecryptRequest.ciphertext_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [RawDecryptRequest.ciphertext_crc32c][google.cloud.kms.v1.RawDecryptRequest.ciphertext_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_ciphertext_crc32c = 4; + + // Integrity verification field. A flag indicating whether + // [RawDecryptRequest.additional_authenticated_data_crc32c][google.cloud.kms.v1.RawDecryptRequest.additional_authenticated_data_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of additional_authenticated_data. A false + // value of this field indicates either that // + // [RawDecryptRequest.additional_authenticated_data_crc32c][google.cloud.kms.v1.RawDecryptRequest.additional_authenticated_data_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [RawDecryptRequest.additional_authenticated_data_crc32c][google.cloud.kms.v1.RawDecryptRequest.additional_authenticated_data_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_additional_authenticated_data_crc32c = 5; + + // Integrity verification field. A flag indicating whether + // [RawDecryptRequest.initialization_vector_crc32c][google.cloud.kms.v1.RawDecryptRequest.initialization_vector_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of initialization_vector. A false value of + // this field indicates either that + // [RawDecryptRequest.initialization_vector_crc32c][google.cloud.kms.v1.RawDecryptRequest.initialization_vector_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [RawDecryptRequest.initialization_vector_crc32c][google.cloud.kms.v1.RawDecryptRequest.initialization_vector_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_initialization_vector_crc32c = 6; +} + +// Response message for +// [KeyManagementService.AsymmetricSign][google.cloud.kms.v1.KeyManagementService.AsymmetricSign]. +message AsymmetricSignResponse { + // The created signature. + bytes signature = 1; + + // Integrity verification field. A CRC32C checksum of the returned + // [AsymmetricSignResponse.signature][google.cloud.kms.v1.AsymmetricSignResponse.signature]. + // An integrity check of + // [AsymmetricSignResponse.signature][google.cloud.kms.v1.AsymmetricSignResponse.signature] + // can be performed by computing the CRC32C checksum of + // [AsymmetricSignResponse.signature][google.cloud.kms.v1.AsymmetricSignResponse.signature] + // and comparing your results to this field. Discard the response in case of + // non-matching checksum values, and perform a limited number of retries. A + // persistent mismatch may indicate an issue in your computation of the CRC32C + // checksum. Note: This field is defined as int64 for reasons of compatibility + // across different languages. However, it is a non-negative integer, which + // will never exceed 2^32-1, and can be safely downconverted to uint32 in + // languages that support this type. + google.protobuf.Int64Value signature_crc32c = 2; + + // Integrity verification field. A flag indicating whether + // [AsymmetricSignRequest.digest_crc32c][google.cloud.kms.v1.AsymmetricSignRequest.digest_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of the + // [digest][google.cloud.kms.v1.AsymmetricSignRequest.digest]. A false value + // of this field indicates either that + // [AsymmetricSignRequest.digest_crc32c][google.cloud.kms.v1.AsymmetricSignRequest.digest_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [AsymmetricSignRequest.digest_crc32c][google.cloud.kms.v1.AsymmetricSignRequest.digest_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_digest_crc32c = 3; + + // The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used for signing. + // Check this field to verify that the intended resource was used for signing. + string name = 4; + + // Integrity verification field. A flag indicating whether + // [AsymmetricSignRequest.data_crc32c][google.cloud.kms.v1.AsymmetricSignRequest.data_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of the + // [data][google.cloud.kms.v1.AsymmetricSignRequest.data]. A false value of + // this field indicates either that + // [AsymmetricSignRequest.data_crc32c][google.cloud.kms.v1.AsymmetricSignRequest.data_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [AsymmetricSignRequest.data_crc32c][google.cloud.kms.v1.AsymmetricSignRequest.data_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_data_crc32c = 5; + + // The [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used for signing. + ProtectionLevel protection_level = 6; +} + +// Response message for +// [KeyManagementService.AsymmetricDecrypt][google.cloud.kms.v1.KeyManagementService.AsymmetricDecrypt]. +message AsymmetricDecryptResponse { + // The decrypted data originally encrypted with the matching public key. + bytes plaintext = 1; + + // Integrity verification field. A CRC32C checksum of the returned + // [AsymmetricDecryptResponse.plaintext][google.cloud.kms.v1.AsymmetricDecryptResponse.plaintext]. + // An integrity check of + // [AsymmetricDecryptResponse.plaintext][google.cloud.kms.v1.AsymmetricDecryptResponse.plaintext] + // can be performed by computing the CRC32C checksum of + // [AsymmetricDecryptResponse.plaintext][google.cloud.kms.v1.AsymmetricDecryptResponse.plaintext] + // and comparing your results to this field. Discard the response in case of + // non-matching checksum values, and perform a limited number of retries. A + // persistent mismatch may indicate an issue in your computation of the CRC32C + // checksum. Note: This field is defined as int64 for reasons of compatibility + // across different languages. However, it is a non-negative integer, which + // will never exceed 2^32-1, and can be safely downconverted to uint32 in + // languages that support this type. + google.protobuf.Int64Value plaintext_crc32c = 2; + + // Integrity verification field. A flag indicating whether + // [AsymmetricDecryptRequest.ciphertext_crc32c][google.cloud.kms.v1.AsymmetricDecryptRequest.ciphertext_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of the + // [ciphertext][google.cloud.kms.v1.AsymmetricDecryptRequest.ciphertext]. A + // false value of this field indicates either that + // [AsymmetricDecryptRequest.ciphertext_crc32c][google.cloud.kms.v1.AsymmetricDecryptRequest.ciphertext_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [AsymmetricDecryptRequest.ciphertext_crc32c][google.cloud.kms.v1.AsymmetricDecryptRequest.ciphertext_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_ciphertext_crc32c = 3; + + // The [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used in + // decryption. + ProtectionLevel protection_level = 4; +} + +// Response message for +// [KeyManagementService.MacSign][google.cloud.kms.v1.KeyManagementService.MacSign]. +message MacSignResponse { + // The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used for signing. + // Check this field to verify that the intended resource was used for signing. + string name = 1; + + // The created signature. + bytes mac = 2; + + // Integrity verification field. A CRC32C checksum of the returned + // [MacSignResponse.mac][google.cloud.kms.v1.MacSignResponse.mac]. An + // integrity check of + // [MacSignResponse.mac][google.cloud.kms.v1.MacSignResponse.mac] can be + // performed by computing the CRC32C checksum of + // [MacSignResponse.mac][google.cloud.kms.v1.MacSignResponse.mac] and + // comparing your results to this field. Discard the response in case of + // non-matching checksum values, and perform a limited number of retries. A + // persistent mismatch may indicate an issue in your computation of the CRC32C + // checksum. Note: This field is defined as int64 for reasons of compatibility + // across different languages. However, it is a non-negative integer, which + // will never exceed 2^32-1, and can be safely downconverted to uint32 in + // languages that support this type. + google.protobuf.Int64Value mac_crc32c = 3; + + // Integrity verification field. A flag indicating whether + // [MacSignRequest.data_crc32c][google.cloud.kms.v1.MacSignRequest.data_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of the + // [data][google.cloud.kms.v1.MacSignRequest.data]. A false value of this + // field indicates either that + // [MacSignRequest.data_crc32c][google.cloud.kms.v1.MacSignRequest.data_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [MacSignRequest.data_crc32c][google.cloud.kms.v1.MacSignRequest.data_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_data_crc32c = 4; + + // The [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used for signing. + ProtectionLevel protection_level = 5; +} + +// Response message for +// [KeyManagementService.MacVerify][google.cloud.kms.v1.KeyManagementService.MacVerify]. +message MacVerifyResponse { + // The resource name of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used for + // verification. Check this field to verify that the intended resource was + // used for verification. + string name = 1; + + // This field indicates whether or not the verification operation for + // [MacVerifyRequest.mac][google.cloud.kms.v1.MacVerifyRequest.mac] over + // [MacVerifyRequest.data][google.cloud.kms.v1.MacVerifyRequest.data] was + // successful. + bool success = 2; + + // Integrity verification field. A flag indicating whether + // [MacVerifyRequest.data_crc32c][google.cloud.kms.v1.MacVerifyRequest.data_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of the + // [data][google.cloud.kms.v1.MacVerifyRequest.data]. A false value of this + // field indicates either that + // [MacVerifyRequest.data_crc32c][google.cloud.kms.v1.MacVerifyRequest.data_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [MacVerifyRequest.data_crc32c][google.cloud.kms.v1.MacVerifyRequest.data_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_data_crc32c = 3; + + // Integrity verification field. A flag indicating whether + // [MacVerifyRequest.mac_crc32c][google.cloud.kms.v1.MacVerifyRequest.mac_crc32c] + // was received by + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService] and used + // for the integrity verification of the + // [data][google.cloud.kms.v1.MacVerifyRequest.mac]. A false value of this + // field indicates either that + // [MacVerifyRequest.mac_crc32c][google.cloud.kms.v1.MacVerifyRequest.mac_crc32c] + // was left unset or that it was not delivered to + // [KeyManagementService][google.cloud.kms.v1.KeyManagementService]. If you've + // set + // [MacVerifyRequest.mac_crc32c][google.cloud.kms.v1.MacVerifyRequest.mac_crc32c] + // but this field is still false, discard the response and perform a limited + // number of retries. + bool verified_mac_crc32c = 4; + + // Integrity verification field. This value is used for the integrity + // verification of [MacVerifyResponse.success]. If the value of this field + // contradicts the value of [MacVerifyResponse.success], discard the response + // and perform a limited number of retries. + bool verified_success_integrity = 5; + + // The [ProtectionLevel][google.cloud.kms.v1.ProtectionLevel] of the + // [CryptoKeyVersion][google.cloud.kms.v1.CryptoKeyVersion] used for + // verification. + ProtectionLevel protection_level = 6; +} + +// Response message for +// [KeyManagementService.GenerateRandomBytes][google.cloud.kms.v1.KeyManagementService.GenerateRandomBytes]. +message GenerateRandomBytesResponse { + // The generated data. + bytes data = 1; + + // Integrity verification field. A CRC32C checksum of the returned + // [GenerateRandomBytesResponse.data][google.cloud.kms.v1.GenerateRandomBytesResponse.data]. + // An integrity check of + // [GenerateRandomBytesResponse.data][google.cloud.kms.v1.GenerateRandomBytesResponse.data] + // can be performed by computing the CRC32C checksum of + // [GenerateRandomBytesResponse.data][google.cloud.kms.v1.GenerateRandomBytesResponse.data] + // and comparing your results to this field. Discard the response in case of + // non-matching checksum values, and perform a limited number of retries. A + // persistent mismatch may indicate an issue in your computation of the CRC32C + // checksum. Note: This field is defined as int64 for reasons of compatibility + // across different languages. However, it is a non-negative integer, which + // will never exceed 2^32-1, and can be safely downconverted to uint32 in + // languages that support this type. + google.protobuf.Int64Value data_crc32c = 3; +} + +// A [Digest][google.cloud.kms.v1.Digest] holds a cryptographic message digest. +message Digest { + // Required. The message digest. + oneof digest { + // A message digest produced with the SHA-256 algorithm. + bytes sha256 = 1; + + // A message digest produced with the SHA-384 algorithm. + bytes sha384 = 2; + + // A message digest produced with the SHA-512 algorithm. + bytes sha512 = 3; + } +} + +// Cloud KMS metadata for the given +// [google.cloud.location.Location][google.cloud.location.Location]. +message LocationMetadata { + // Indicates whether [CryptoKeys][google.cloud.kms.v1.CryptoKey] with + // [protection_level][google.cloud.kms.v1.CryptoKeyVersionTemplate.protection_level] + // [HSM][google.cloud.kms.v1.ProtectionLevel.HSM] can be created in this + // location. + bool hsm_available = 1; + + // Indicates whether [CryptoKeys][google.cloud.kms.v1.CryptoKey] with + // [protection_level][google.cloud.kms.v1.CryptoKeyVersionTemplate.protection_level] + // [EXTERNAL][google.cloud.kms.v1.ProtectionLevel.EXTERNAL] can be created in + // this location. + bool ekm_available = 2; +} diff --git a/proto/google/cloud/resourcemanager/v3/BUILD b/proto/google/cloud/resourcemanager/v3/BUILD new file mode 100644 index 0000000..b5296c9 --- /dev/null +++ b/proto/google/cloud/resourcemanager/v3/BUILD @@ -0,0 +1,23 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "resourcemanager_proto", + srcs = [ + "folders.proto", + "organizations.proto", + "projects.proto", + "tag_holds.proto", + "tag_keys.proto", + "tag_values.proto", + ], + strip_import_prefix = "/proto/", + deps = [ + "//proto/google:com_google_protobuf", + "//proto/google:googleapis", + "//proto/google/iam/v1:iam_policy_proto", + "//proto/google/iam/v1:policy_proto", + "//proto/google/longrunning:operations_proto", + ], +) diff --git a/proto/google/cloud/resourcemanager/v3/cloudresourcemanager_v3.yaml b/proto/google/cloud/resourcemanager/v3/cloudresourcemanager_v3.yaml new file mode 100644 index 0000000..a1c8af2 --- /dev/null +++ b/proto/google/cloud/resourcemanager/v3/cloudresourcemanager_v3.yaml @@ -0,0 +1,214 @@ +type: google.api.Service +config_version: 3 +name: cloudresourcemanager.googleapis.com +title: Cloud Resource Manager API + +apis: +- name: google.cloud.resourcemanager.v3.Folders +- name: google.cloud.resourcemanager.v3.Organizations +- name: google.cloud.resourcemanager.v3.Projects +- name: google.cloud.resourcemanager.v3.TagBindings +- name: google.cloud.resourcemanager.v3.TagHolds +- name: google.cloud.resourcemanager.v3.TagKeys +- name: google.cloud.resourcemanager.v3.TagValues +- name: google.longrunning.Operations + +types: +- name: google.cloud.resourcemanager.v3.CreateFolderMetadata +- name: google.cloud.resourcemanager.v3.CreateProjectMetadata +- name: google.cloud.resourcemanager.v3.CreateTagBindingMetadata +- name: google.cloud.resourcemanager.v3.CreateTagKeyMetadata +- name: google.cloud.resourcemanager.v3.CreateTagValueMetadata +- name: google.cloud.resourcemanager.v3.DeleteFolderMetadata +- name: google.cloud.resourcemanager.v3.DeleteOrganizationMetadata +- name: google.cloud.resourcemanager.v3.DeleteProjectMetadata +- name: google.cloud.resourcemanager.v3.DeleteTagBindingMetadata +- name: google.cloud.resourcemanager.v3.DeleteTagKeyMetadata +- name: google.cloud.resourcemanager.v3.DeleteTagValueMetadata +- name: google.cloud.resourcemanager.v3.MoveFolderMetadata +- name: google.cloud.resourcemanager.v3.MoveProjectMetadata +- name: google.cloud.resourcemanager.v3.UndeleteFolderMetadata +- name: google.cloud.resourcemanager.v3.UndeleteOrganizationMetadata +- name: google.cloud.resourcemanager.v3.UndeleteProjectMetadata +- name: google.cloud.resourcemanager.v3.UpdateFolderMetadata +- name: google.cloud.resourcemanager.v3.UpdateProjectMetadata +- name: google.cloud.resourcemanager.v3.UpdateTagKeyMetadata +- name: google.cloud.resourcemanager.v3.UpdateTagValueMetadata + +documentation: + summary: |- + Creates, reads, and updates metadata for Google Cloud Platform resource + containers. + +backend: + rules: + - selector: 'google.cloud.resourcemanager.v3.Folders.*' + deadline: 30.0 + - selector: 'google.cloud.resourcemanager.v3.Organizations.*' + deadline: 30.0 + - selector: 'google.cloud.resourcemanager.v3.Projects.*' + deadline: 30.0 + - selector: 'google.cloud.resourcemanager.v3.TagBindings.*' + deadline: 30.0 + - selector: 'google.cloud.resourcemanager.v3.TagHolds.*' + deadline: 30.0 + - selector: 'google.cloud.resourcemanager.v3.TagKeys.*' + deadline: 30.0 + - selector: 'google.cloud.resourcemanager.v3.TagValues.*' + deadline: 30.0 + - selector: google.longrunning.Operations.GetOperation + deadline: 30.0 + +http: + rules: + - selector: google.longrunning.Operations.GetOperation + get: '/v3/{name=operations/**}' + +authentication: + rules: + - selector: 'google.cloud.resourcemanager.v3.Folders.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.cloud.resourcemanager.v3.Folders.GetFolder + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.Folders.GetIamPolicy + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.Folders.ListFolders + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.Folders.SearchFolders + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: 'google.cloud.resourcemanager.v3.Organizations.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.Organizations.SetIamPolicy + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: 'google.cloud.resourcemanager.v3.Projects.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.cloud.resourcemanager.v3.Projects.GetIamPolicy + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.Projects.GetProject + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.Projects.ListProjects + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.Projects.SearchProjects + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.Projects.TestIamPermissions + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.TagBindings.CreateTagBinding + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.cloud.resourcemanager.v3.TagBindings.DeleteTagBinding + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.cloud.resourcemanager.v3.TagBindings.ListEffectiveTags + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.TagBindings.ListTagBindings + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.TagHolds.CreateTagHold + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.cloud.resourcemanager.v3.TagHolds.DeleteTagHold + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.cloud.resourcemanager.v3.TagHolds.ListTagHolds + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: 'google.cloud.resourcemanager.v3.TagKeys.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.cloud.resourcemanager.v3.TagKeys.GetIamPolicy + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.TagKeys.GetNamespacedTagKey + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.TagKeys.GetTagKey + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.TagKeys.ListTagKeys + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: 'google.cloud.resourcemanager.v3.TagValues.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.cloud.resourcemanager.v3.TagValues.GetIamPolicy + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.TagValues.GetNamespacedTagValue + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.TagValues.GetTagValue + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.resourcemanager.v3.TagValues.ListTagValues + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.longrunning.Operations.GetOperation + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only diff --git a/proto/google/cloud/resourcemanager/v3/cloudresourcemanager_v3_grpc_service_config.json b/proto/google/cloud/resourcemanager/v3/cloudresourcemanager_v3_grpc_service_config.json new file mode 100644 index 0000000..477caca --- /dev/null +++ b/proto/google/cloud/resourcemanager/v3/cloudresourcemanager_v3_grpc_service_config.json @@ -0,0 +1,187 @@ +{ + "methodConfig": [ + { + "name": [ + { + "service": "google.cloud.resourcemanager.v3.Projects", + "method": "ListProjects" + }, + { + "service": "google.cloud.resourcemanager.v3.Projects", + "method": "GetProject" + }, + { + "service": "google.cloud.resourcemanager.v3.Projects", + "method": "GetIamPolicy" + }, + { + "service": "google.cloud.resourcemanager.v3.Folders", + "method": "ListFolders" + }, + { + "service": "google.cloud.resourcemanager.v3.Folders", + "method": "GetFolder" + }, + { + "service": "google.cloud.resourcemanager.v3.Folders", + "method": "GetIamPolicy" + }, + { + "service": "google.cloud.resourcemanager.v3.Organizations", + "method": "GetOrganization" + }, + { + "service": "google.cloud.resourcemanager.v3.Organizations", + "method": "GetIamPolicy" + }, + { + "service": "google.cloud.resourcemanager.v3.TagKeys", + "method": "ListTagKeys" + }, + { + "service": "google.cloud.resourcemanager.v3.TagKeys", + "method": "GetTagKey" + }, + { + "service": "google.cloud.resourcemanager.v3.TagKeys", + "method": "GetIamPolicy" + }, + { + "service": "google.cloud.resourcemanager.v3.TagValues", + "method": "ListTagValues" + }, + { + "service": "google.cloud.resourcemanager.v3.TagValues", + "method": "GetTagValue" + }, + { + "service": "google.cloud.resourcemanager.v3.TagValues", + "method": "GetIamPolicy" + }, + { + "service": "google.cloud.resourcemanager.v3.TagBindings", + "method": "ListTagBindings" + } + ], + "timeout": "60s", + "retryPolicy": { + "maxAttempts": 5, + "initialBackoff": "0.100s", + "maxBackoff": "60s", + "backoffMultiplier": 1.3, + "retryableStatusCodes": [ + "UNAVAILABLE" + ] + } + }, + { + "name": [ + { + "service": "google.cloud.resourcemanager.v3.Projects", + "method": "CreateProject" + }, + { + "service": "google.cloud.resourcemanager.v3.Projects", + "method": "DeleteProject" + }, + { + "service": "google.cloud.resourcemanager.v3.Projects", + "method": "UndeleteProject" + }, + { + "service": "google.cloud.resourcemanager.v3.Projects", + "method": "UpdateProject" + }, + { + "service": "google.cloud.resourcemanager.v3.Projects", + "method": "MoveProject" + }, + { + "service": "google.cloud.resourcemanager.v3.Projects", + "method": "SetIamPolicy" + }, + { + "service": "google.cloud.resourcemanager.v3.Projects", + "method": "SearchProjects" + }, + { + "service": "google.cloud.resourcemanager.v3.Folders", + "method": "CreateFolder" + }, + { + "service": "google.cloud.resourcemanager.v3.Folders", + "method": "DeleteFolder" + }, + { + "service": "google.cloud.resourcemanager.v3.Folders", + "method": "UndeleteFolder" + }, + { + "service": "google.cloud.resourcemanager.v3.Folders", + "method": "UpdateFolder" + }, + { + "service": "google.cloud.resourcemanager.v3.Folders", + "method": "MoveFolder" + }, + { + "service": "google.cloud.resourcemanager.v3.Folders", + "method": "SetIamPolicy" + }, + { + "service": "google.cloud.resourcemanager.v3.Folders", + "method": "SearchFolders" + }, + { + "service": "google.cloud.resourcemanager.v3.Organizations", + "method": "SearchOrganizations" + }, + { + "service": "google.cloud.resourcemanager.v3.Organizations", + "method": "SetIamPolicy" + }, + { + "service": "google.cloud.resourcemanager.v3.TagKeys", + "method": "CreateTagKey" + }, + { + "service": "google.cloud.resourcemanager.v3.TagKeys", + "method": "UpdateTagKey" + }, + { + "service": "google.cloud.resourcemanager.v3.TagKeys", + "method": "DeleteTagKey" + }, + { + "service": "google.cloud.resourcemanager.v3.TagKeys", + "method": "SetIamPolicy" + }, + { + "service": "google.cloud.resourcemanager.v3.TagValues", + "method": "CreateTagValue" + }, + { + "service": "google.cloud.resourcemanager.v3.TagValues", + "method": "UpdateTagValue" + }, + { + "service": "google.cloud.resourcemanager.v3.TagValues", + "method": "DeleteTagValue" + }, + { + "service": "google.cloud.resourcemanager.v3.TagValues", + "method": "SetIamPolicy" + }, + { + "service": "google.cloud.resourcemanager.v3.TagBindings", + "method": "CreateTagBinding" + }, + { + "service": "google.cloud.resourcemanager.v3.TagBindings", + "method": "DeleteTagBinding" + } + ], + "timeout": "60s" + } + ] +} diff --git a/proto/google/cloud/resourcemanager/v3/folders.proto b/proto/google/cloud/resourcemanager/v3/folders.proto new file mode 100644 index 0000000..9f9e012 --- /dev/null +++ b/proto/google/cloud/resourcemanager/v3/folders.proto @@ -0,0 +1,546 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.resourcemanager.v3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/iam/v1/iam_policy.proto"; +import "google/iam/v1/policy.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ResourceManager.V3"; +option go_package = "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb;resourcemanagerpb"; +option java_multiple_files = true; +option java_outer_classname = "FoldersProto"; +option java_package = "com.google.cloud.resourcemanager.v3"; +option php_namespace = "Google\\Cloud\\ResourceManager\\V3"; +option ruby_package = "Google::Cloud::ResourceManager::V3"; + +// Manages Cloud Platform folder resources. +// Folders can be used to organize the resources under an +// organization and to control the policies applied to groups of resources. +service Folders { + option (google.api.default_host) = "cloudresourcemanager.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/cloud-platform.read-only"; + + // Retrieves a folder identified by the supplied resource name. + // Valid folder resource names have the format `folders/{folder_id}` + // (for example, `folders/1234`). + // The caller must have `resourcemanager.folders.get` permission on the + // identified folder. + rpc GetFolder(GetFolderRequest) returns (Folder) { + option (google.api.http) = { + get: "/v3/{name=folders/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Lists the folders that are direct descendants of supplied parent resource. + // `list()` provides a strongly consistent view of the folders underneath + // the specified parent resource. + // `list()` returns folders sorted based upon the (ascending) lexical ordering + // of their display_name. + // The caller must have `resourcemanager.folders.list` permission on the + // identified parent. + rpc ListFolders(ListFoldersRequest) returns (ListFoldersResponse) { + option (google.api.http) = { + get: "/v3/folders" + }; + option (google.api.method_signature) = "parent"; + } + + // Search for folders that match specific filter criteria. + // `search()` provides an eventually consistent view of the folders a user has + // access to which meet the specified filter criteria. + // + // This will only return folders on which the caller has the + // permission `resourcemanager.folders.get`. + rpc SearchFolders(SearchFoldersRequest) returns (SearchFoldersResponse) { + option (google.api.http) = { + get: "/v3/folders:search" + }; + option (google.api.method_signature) = "query"; + } + + // Creates a folder in the resource hierarchy. + // Returns an `Operation` which can be used to track the progress of the + // folder creation workflow. + // Upon success, the `Operation.response` field will be populated with the + // created Folder. + // + // In order to succeed, the addition of this new folder must not violate + // the folder naming, height, or fanout constraints. + // + // + The folder's `display_name` must be distinct from all other folders that + // share its parent. + // + The addition of the folder must not cause the active folder hierarchy + // to exceed a height of 10. Note, the full active + deleted folder hierarchy + // is allowed to reach a height of 20; this provides additional headroom when + // moving folders that contain deleted folders. + // + The addition of the folder must not cause the total number of folders + // under its parent to exceed 300. + // + // If the operation fails due to a folder constraint violation, some errors + // may be returned by the `CreateFolder` request, with status code + // `FAILED_PRECONDITION` and an error description. Other folder constraint + // violations will be communicated in the `Operation`, with the specific + // `PreconditionFailure` returned in the details list in the `Operation.error` + // field. + // + // The caller must have `resourcemanager.folders.create` permission on the + // identified parent. + rpc CreateFolder(CreateFolderRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v3/folders" + body: "folder" + }; + option (google.api.method_signature) = "folder"; + option (google.longrunning.operation_info) = { + response_type: "Folder" + metadata_type: "CreateFolderMetadata" + }; + } + + // Updates a folder, changing its `display_name`. + // Changes to the folder `display_name` will be rejected if they violate + // either the `display_name` formatting rules or the naming constraints + // described in the + // [CreateFolder][google.cloud.resourcemanager.v3.Folders.CreateFolder] + // documentation. + // + // The folder's `display_name` must start and end with a letter or digit, + // may contain letters, digits, spaces, hyphens and underscores and can be + // between 3 and 30 characters. This is captured by the regular expression: + // `[\p{L}\p{N}][\p{L}\p{N}_- ]{1,28}[\p{L}\p{N}]`. + // The caller must have `resourcemanager.folders.update` permission on the + // identified folder. + // + // If the update fails due to the unique name constraint then a + // `PreconditionFailure` explaining this violation will be returned + // in the Status.details field. + rpc UpdateFolder(UpdateFolderRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { + patch: "/v3/{folder.name=folders/*}" + body: "folder" + }; + option (google.api.method_signature) = "folder,update_mask"; + option (google.longrunning.operation_info) = { + response_type: "Folder" + metadata_type: "UpdateFolderMetadata" + }; + } + + // Moves a folder under a new resource parent. + // Returns an `Operation` which can be used to track the progress of the + // folder move workflow. + // Upon success, the `Operation.response` field will be populated with the + // moved folder. + // Upon failure, a `FolderOperationError` categorizing the failure cause will + // be returned - if the failure occurs synchronously then the + // `FolderOperationError` will be returned in the `Status.details` field. + // If it occurs asynchronously, then the FolderOperation will be returned + // in the `Operation.error` field. + // In addition, the `Operation.metadata` field will be populated with a + // `FolderOperation` message as an aid to stateless clients. + // Folder moves will be rejected if they violate either the naming, height, + // or fanout constraints described in the + // [CreateFolder][google.cloud.resourcemanager.v3.Folders.CreateFolder] + // documentation. The caller must have `resourcemanager.folders.move` + // permission on the folder's current and proposed new parent. + rpc MoveFolder(MoveFolderRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v3/{name=folders/*}:move" + body: "*" + }; + option (google.api.method_signature) = "name,destination_parent"; + option (google.longrunning.operation_info) = { + response_type: "Folder" + metadata_type: "MoveFolderMetadata" + }; + } + + // Requests deletion of a folder. The folder is moved into the + // [DELETE_REQUESTED][google.cloud.resourcemanager.v3.Folder.State.DELETE_REQUESTED] + // state immediately, and is deleted approximately 30 days later. This method + // may only be called on an empty folder, where a folder is empty if it + // doesn't contain any folders or projects in the + // [ACTIVE][google.cloud.resourcemanager.v3.Folder.State.ACTIVE] state. If + // called on a folder in + // [DELETE_REQUESTED][google.cloud.resourcemanager.v3.Folder.State.DELETE_REQUESTED] + // state the operation will result in a no-op success. + // The caller must have `resourcemanager.folders.delete` permission on the + // identified folder. + rpc DeleteFolder(DeleteFolderRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v3/{name=folders/*}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "Folder" + metadata_type: "DeleteFolderMetadata" + }; + } + + // Cancels the deletion request for a folder. This method may be called on a + // folder in any state. If the folder is in the + // [ACTIVE][google.cloud.resourcemanager.v3.Folder.State.ACTIVE] state the + // result will be a no-op success. In order to succeed, the folder's parent + // must be in the + // [ACTIVE][google.cloud.resourcemanager.v3.Folder.State.ACTIVE] state. In + // addition, reintroducing the folder into the tree must not violate folder + // naming, height, and fanout constraints described in the + // [CreateFolder][google.cloud.resourcemanager.v3.Folders.CreateFolder] + // documentation. The caller must have `resourcemanager.folders.undelete` + // permission on the identified folder. + rpc UndeleteFolder(UndeleteFolderRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v3/{name=folders/*}:undelete" + body: "*" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "Folder" + metadata_type: "UndeleteFolderMetadata" + }; + } + + // Gets the access control policy for a folder. The returned policy may be + // empty if no such policy or resource exists. The `resource` field should + // be the folder's resource name, for example: "folders/1234". + // The caller must have `resourcemanager.folders.getIamPolicy` permission + // on the identified folder. + rpc GetIamPolicy(google.iam.v1.GetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + post: "/v3/{resource=folders/*}:getIamPolicy" + body: "*" + }; + option (google.api.method_signature) = "resource"; + } + + // Sets the access control policy on a folder, replacing any existing policy. + // The `resource` field should be the folder's resource name, for example: + // "folders/1234". + // The caller must have `resourcemanager.folders.setIamPolicy` permission + // on the identified folder. + rpc SetIamPolicy(google.iam.v1.SetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + post: "/v3/{resource=folders/*}:setIamPolicy" + body: "*" + }; + option (google.api.method_signature) = "resource,policy"; + } + + // Returns permissions that a caller has on the specified folder. + // The `resource` field should be the folder's resource name, + // for example: "folders/1234". + // + // There are no permissions required for making this API call. + rpc TestIamPermissions(google.iam.v1.TestIamPermissionsRequest) + returns (google.iam.v1.TestIamPermissionsResponse) { + option (google.api.http) = { + post: "/v3/{resource=folders/*}:testIamPermissions" + body: "*" + }; + option (google.api.method_signature) = "resource,permissions"; + } +} + +// A folder in an organization's resource hierarchy, used to +// organize that organization's resources. +message Folder { + option (google.api.resource) = { + type: "cloudresourcemanager.googleapis.com/Folder" + pattern: "folders/{folder}" + style: DECLARATIVE_FRIENDLY + }; + + // Folder lifecycle states. + enum State { + // Unspecified state. + STATE_UNSPECIFIED = 0; + + // The normal and active state. + ACTIVE = 1; + + // The folder has been marked for deletion by the user. + DELETE_REQUESTED = 2; + } + + // Output only. The resource name of the folder. + // Its format is `folders/{folder_id}`, for example: "folders/1234". + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Required. The folder's parent's resource name. + // Updates to the folder's parent must be performed using + // [MoveFolder][google.cloud.resourcemanager.v3.Folders.MoveFolder]. + string parent = 2 [(google.api.field_behavior) = REQUIRED]; + + // The folder's display name. + // A folder's display name must be unique amongst its siblings. For example, + // no two folders with the same parent can share the same display name. + // The display name must start and end with a letter or digit, may contain + // letters, digits, spaces, hyphens and underscores and can be no longer + // than 30 characters. This is captured by the regular expression: + // `[\p{L}\p{N}]([\p{L}\p{N}_- ]{0,28}[\p{L}\p{N}])?`. + string display_name = 3; + + // Output only. The lifecycle state of the folder. + // Updates to the state must be performed using + // [DeleteFolder][google.cloud.resourcemanager.v3.Folders.DeleteFolder] and + // [UndeleteFolder][google.cloud.resourcemanager.v3.Folders.UndeleteFolder]. + State state = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Timestamp when the folder was created. + google.protobuf.Timestamp create_time = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Timestamp when the folder was last modified. + google.protobuf.Timestamp update_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Timestamp when the folder was requested to be deleted. + google.protobuf.Timestamp delete_time = 7 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. A checksum computed by the server based on the current value + // of the folder resource. This may be sent on update and delete requests to + // ensure the client has an up-to-date value before proceeding. + string etag = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// The GetFolder request message. +message GetFolderRequest { + // Required. The resource name of the folder to retrieve. + // Must be of the form `folders/{folder_id}`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Folder" + } + ]; +} + +// The ListFolders request message. +message ListFoldersRequest { + // Required. The name of the parent resource whose folders are being listed. + // Only children of this parent resource are listed; descendants are not + // listed. + // + // If the parent is a folder, use the value `folders/{folder_id}`. If the + // parent is an organization, use the value `organizations/{org_id}`. + // + // Access to this method is controlled by checking the + // `resourcemanager.folders.list` permission on the `parent`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { child_type: "*" } + ]; + + // Optional. The maximum number of folders to return in the response. The + // server can return fewer folders than requested. If unspecified, server + // picks an appropriate default. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to `ListFolders` + // that indicates where this listing should continue from. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Controls whether folders in the + // [DELETE_REQUESTED][google.cloud.resourcemanager.v3.Folder.State.DELETE_REQUESTED] + // state should be returned. Defaults to false. + bool show_deleted = 4 [(google.api.field_behavior) = OPTIONAL]; +} + +// The ListFolders response message. +message ListFoldersResponse { + // A possibly paginated list of folders that are direct descendants of + // the specified parent resource. + repeated Folder folders = 1; + + // A pagination token returned from a previous call to `ListFolders` + // that indicates from where listing should continue. + string next_page_token = 2; +} + +// The request message for searching folders. +message SearchFoldersRequest { + // Optional. The maximum number of folders to return in the response. The + // server can return fewer folders than requested. If unspecified, server + // picks an appropriate default. + int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to + // `SearchFolders` that indicates from where search should continue. + string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Search criteria used to select the folders to return. + // If no search criteria is specified then all accessible folders will be + // returned. + // + // Query expressions can be used to restrict results based upon displayName, + // state and parent, where the operators `=` (`:`) `NOT`, `AND` and `OR` + // can be used along with the suffix wildcard symbol `*`. + // + // The `displayName` field in a query expression should use escaped quotes + // for values that include whitespace to prevent unexpected behavior. + // + // ``` + // | Field | Description | + // |-------------------------|----------------------------------------| + // | displayName | Filters by displayName. | + // | parent | Filters by parent (for example: folders/123). | + // | state, lifecycleState | Filters by state. | + // ``` + // + // Some example queries are: + // + // * Query `displayName=Test*` returns Folder resources whose display name + // starts with "Test". + // * Query `state=ACTIVE` returns Folder resources with + // `state` set to `ACTIVE`. + // * Query `parent=folders/123` returns Folder resources that have + // `folders/123` as a parent resource. + // * Query `parent=folders/123 AND state=ACTIVE` returns active + // Folder resources that have `folders/123` as a parent resource. + // * Query `displayName=\\"Test String\\"` returns Folder resources with + // display names that include both "Test" and "String". + string query = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// The response message for searching folders. +message SearchFoldersResponse { + // A possibly paginated folder search results. + // the specified parent resource. + repeated Folder folders = 1; + + // A pagination token returned from a previous call to `SearchFolders` + // that indicates from where searching should continue. + string next_page_token = 2; +} + +// The CreateFolder request message. +message CreateFolderRequest { + // Required. The folder being created, only the display name and parent will + // be consulted. All other fields will be ignored. + Folder folder = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// Metadata pertaining to the Folder creation process. +message CreateFolderMetadata { + // The display name of the folder. + string display_name = 1; + + // The resource name of the folder or organization we are creating the folder + // under. + string parent = 2; +} + +// The request sent to the +// [UpdateFolder][google.cloud.resourcemanager.v3.Folder.UpdateFolder] +// method. +// +// Only the `display_name` field can be changed. All other fields will be +// ignored. Use the +// [MoveFolder][google.cloud.resourcemanager.v3.Folders.MoveFolder] method to +// change the `parent` field. +message UpdateFolderRequest { + // Required. The new definition of the Folder. It must include the `name` + // field, which cannot be changed. + Folder folder = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. Fields to be updated. + // Only the `display_name` can be updated. + google.protobuf.FieldMask update_mask = 2 + [(google.api.field_behavior) = REQUIRED]; +} + +// A status object which is used as the `metadata` field for the Operation +// returned by UpdateFolder. +message UpdateFolderMetadata {} + +// The MoveFolder request message. +message MoveFolderRequest { + // Required. The resource name of the Folder to move. + // Must be of the form folders/{folder_id} + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Folder" + } + ]; + + // Required. The resource name of the folder or organization which should be + // the folder's new parent. Must be of the form `folders/{folder_id}` or + // `organizations/{org_id}`. + string destination_parent = 2 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { child_type: "*" } + ]; +} + +// Metadata pertaining to the folder move process. +message MoveFolderMetadata { + // The display name of the folder. + string display_name = 1; + + // The resource name of the folder's parent. + string source_parent = 2; + + // The resource name of the folder or organization to move the folder to. + string destination_parent = 3; +} + +// The DeleteFolder request message. +message DeleteFolderRequest { + // Required. The resource name of the folder to be deleted. + // Must be of the form `folders/{folder_id}`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Folder" + } + ]; +} + +// A status object which is used as the `metadata` field for the `Operation` +// returned by `DeleteFolder`. +message DeleteFolderMetadata {} + +// The UndeleteFolder request message. +message UndeleteFolderRequest { + // Required. The resource name of the folder to undelete. + // Must be of the form `folders/{folder_id}`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Folder" + } + ]; +} + +// A status object which is used as the `metadata` field for the `Operation` +// returned by `UndeleteFolder`. +message UndeleteFolderMetadata {} diff --git a/proto/google/cloud/resourcemanager/v3/organizations.proto b/proto/google/cloud/resourcemanager/v3/organizations.proto new file mode 100644 index 0000000..64c6276 --- /dev/null +++ b/proto/google/cloud/resourcemanager/v3/organizations.proto @@ -0,0 +1,241 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.resourcemanager.v3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/iam/v1/iam_policy.proto"; +import "google/iam/v1/policy.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ResourceManager.V3"; +option go_package = "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb;resourcemanagerpb"; +option java_multiple_files = true; +option java_outer_classname = "OrganizationsProto"; +option java_package = "com.google.cloud.resourcemanager.v3"; +option php_namespace = "Google\\Cloud\\ResourceManager\\V3"; +option ruby_package = "Google::Cloud::ResourceManager::V3"; + +// Allows users to manage their organization resources. +service Organizations { + option (google.api.default_host) = "cloudresourcemanager.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/cloud-platform.read-only"; + + // Fetches an organization resource identified by the specified resource name. + rpc GetOrganization(GetOrganizationRequest) returns (Organization) { + option (google.api.http) = { + get: "/v3/{name=organizations/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Searches organization resources that are visible to the user and satisfy + // the specified filter. This method returns organizations in an unspecified + // order. New organizations do not necessarily appear at the end of the + // results, and may take a small amount of time to appear. + // + // Search will only return organizations on which the user has the permission + // `resourcemanager.organizations.get` + rpc SearchOrganizations(SearchOrganizationsRequest) + returns (SearchOrganizationsResponse) { + option (google.api.http) = { + get: "/v3/organizations:search" + }; + option (google.api.method_signature) = "query"; + } + + // Gets the access control policy for an organization resource. The policy may + // be empty if no such policy or resource exists. The `resource` field should + // be the organization's resource name, for example: "organizations/123". + // + // Authorization requires the IAM permission + // `resourcemanager.organizations.getIamPolicy` on the specified organization. + rpc GetIamPolicy(google.iam.v1.GetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + post: "/v3/{resource=organizations/*}:getIamPolicy" + body: "*" + }; + option (google.api.method_signature) = "resource"; + } + + // Sets the access control policy on an organization resource. Replaces any + // existing policy. The `resource` field should be the organization's resource + // name, for example: "organizations/123". + // + // Authorization requires the IAM permission + // `resourcemanager.organizations.setIamPolicy` on the specified organization. + rpc SetIamPolicy(google.iam.v1.SetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + post: "/v3/{resource=organizations/*}:setIamPolicy" + body: "*" + }; + option (google.api.method_signature) = "resource"; + } + + // Returns the permissions that a caller has on the specified organization. + // The `resource` field should be the organization's resource name, + // for example: "organizations/123". + // + // There are no permissions required for making this API call. + rpc TestIamPermissions(google.iam.v1.TestIamPermissionsRequest) + returns (google.iam.v1.TestIamPermissionsResponse) { + option (google.api.http) = { + post: "/v3/{resource=organizations/*}:testIamPermissions" + body: "*" + }; + option (google.api.method_signature) = "resource,permissions"; + } +} + +// The root node in the resource hierarchy to which a particular entity's +// (a company, for example) resources belong. +message Organization { + option (google.api.resource) = { + type: "cloudresourcemanager.googleapis.com/Organization" + pattern: "organizations/{organization}" + style: DECLARATIVE_FRIENDLY + }; + + // Organization lifecycle states. + enum State { + // Unspecified state. This is only useful for distinguishing unset values. + STATE_UNSPECIFIED = 0; + + // The normal and active state. + ACTIVE = 1; + + // The organization has been marked for deletion by the user. + DELETE_REQUESTED = 2; + } + + // Output only. The resource name of the organization. This is the + // organization's relative path in the API. Its format is + // "organizations/[organization_id]". For example, "organizations/1234". + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. A human-readable string that refers to the organization in the + // Google Cloud Console. This string is set by the server and cannot be + // changed. The string will be set to the primary domain (for example, + // "google.com") of the Google Workspace customer that owns the organization. + string display_name = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // The owner of this organization. The owner should be specified on + // creation. Once set, it cannot be changed. + // + // The lifetime of the organization and all of its descendants are bound to + // the owner. If the owner is deleted, the organization and all its + // descendants will be deleted. + oneof owner { + // Immutable. The G Suite / Workspace customer id used in the Directory API. + string directory_customer_id = 3 [(google.api.field_behavior) = IMMUTABLE]; + } + + // Output only. The organization's current lifecycle state. + State state = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Timestamp when the Organization was created. + google.protobuf.Timestamp create_time = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Timestamp when the Organization was last modified. + google.protobuf.Timestamp update_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Timestamp when the Organization was requested for deletion. + google.protobuf.Timestamp delete_time = 7 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. A checksum computed by the server based on the current value + // of the Organization resource. This may be sent on update and delete + // requests to ensure the client has an up-to-date value before proceeding. + string etag = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// The request sent to the `GetOrganization` method. The `name` field is +// required. `organization_id` is no longer accepted. +message GetOrganizationRequest { + // Required. The resource name of the Organization to fetch. This is the + // organization's relative path in the API, formatted as + // "organizations/[organizationId]". For example, "organizations/1234". + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Organization" + } + ]; +} + +// The request sent to the `SearchOrganizations` method. +message SearchOrganizationsRequest { + // Optional. The maximum number of organizations to return in the response. + // The server can return fewer organizations than requested. If unspecified, + // server picks an appropriate default. + int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to + // `SearchOrganizations` that indicates from where listing should continue. + string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An optional query string used to filter the Organizations to + // return in the response. Query rules are case-insensitive. + // + // + // ``` + // | Field | Description | + // |------------------|--------------------------------------------| + // | directoryCustomerId, owner.directoryCustomerId | Filters by directory + // customer id. | + // | domain | Filters by domain. | + // ``` + // + // Organizations may be queried by `directoryCustomerId` or by + // `domain`, where the domain is a G Suite domain, for example: + // + // * Query `directorycustomerid:123456789` returns Organization + // resources with `owner.directory_customer_id` equal to `123456789`. + // * Query `domain:google.com` returns Organization resources corresponding + // to the domain `google.com`. + string query = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// The response returned from the `SearchOrganizations` method. +message SearchOrganizationsResponse { + // The list of Organizations that matched the search query, possibly + // paginated. + repeated Organization organizations = 1; + + // A pagination token to be used to retrieve the next page of results. If the + // result is too large to fit within the page size specified in the request, + // this field will be set with a token that can be used to fetch the next page + // of results. If this field is empty, it indicates that this response + // contains the last page of results. + string next_page_token = 2; +} + +// A status object which is used as the `metadata` field for the operation +// returned by DeleteOrganization. +message DeleteOrganizationMetadata {} + +// A status object which is used as the `metadata` field for the Operation +// returned by UndeleteOrganization. +message UndeleteOrganizationMetadata {} diff --git a/proto/google/cloud/resourcemanager/v3/projects.proto b/proto/google/cloud/resourcemanager/v3/projects.proto new file mode 100644 index 0000000..60264e1 --- /dev/null +++ b/proto/google/cloud/resourcemanager/v3/projects.proto @@ -0,0 +1,638 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.resourcemanager.v3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/iam/v1/iam_policy.proto"; +import "google/iam/v1/policy.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ResourceManager.V3"; +option go_package = "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb;resourcemanagerpb"; +option java_multiple_files = true; +option java_outer_classname = "ProjectsProto"; +option java_package = "com.google.cloud.resourcemanager.v3"; +option php_namespace = "Google\\Cloud\\ResourceManager\\V3"; +option ruby_package = "Google::Cloud::ResourceManager::V3"; + +// Manages Google Cloud Projects. +service Projects { + option (google.api.default_host) = "cloudresourcemanager.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/cloud-platform.read-only"; + + // Retrieves the project identified by the specified `name` (for example, + // `projects/415104041262`). + // + // The caller must have `resourcemanager.projects.get` permission + // for this project. + rpc GetProject(GetProjectRequest) returns (Project) { + option (google.api.http) = { + get: "/v3/{name=projects/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Lists projects that are direct children of the specified folder or + // organization resource. `list()` provides a strongly consistent view of the + // projects underneath the specified parent resource. `list()` returns + // projects sorted based upon the (ascending) lexical ordering of their + // `display_name`. The caller must have `resourcemanager.projects.list` + // permission on the identified parent. + rpc ListProjects(ListProjectsRequest) returns (ListProjectsResponse) { + option (google.api.http) = { + get: "/v3/projects" + }; + option (google.api.method_signature) = "parent"; + } + + // Search for projects that the caller has both `resourcemanager.projects.get` + // permission on, and also satisfy the specified query. + // + // This method returns projects in an unspecified order. + // + // This method is eventually consistent with project mutations; this means + // that a newly created project may not appear in the results or recent + // updates to an existing project may not be reflected in the results. To + // retrieve the latest state of a project, use the + // [GetProject][google.cloud.resourcemanager.v3.Projects.GetProject] method. + rpc SearchProjects(SearchProjectsRequest) returns (SearchProjectsResponse) { + option (google.api.http) = { + get: "/v3/projects:search" + }; + option (google.api.method_signature) = "query"; + } + + // Request that a new project be created. The result is an `Operation` which + // can be used to track the creation process. This process usually takes a few + // seconds, but can sometimes take much longer. The tracking `Operation` is + // automatically deleted after a few hours, so there is no need to call + // `DeleteOperation`. + rpc CreateProject(CreateProjectRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v3/projects" + body: "project" + }; + option (google.api.method_signature) = "project"; + option (google.longrunning.operation_info) = { + response_type: "Project" + metadata_type: "CreateProjectMetadata" + }; + } + + // Updates the `display_name` and labels of the project identified by the + // specified `name` (for example, `projects/415104041262`). Deleting all + // labels requires an update mask for labels field. + // + // The caller must have `resourcemanager.projects.update` permission for this + // project. + rpc UpdateProject(UpdateProjectRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + patch: "/v3/{project.name=projects/*}" + body: "project" + }; + option (google.api.method_signature) = "project,update_mask"; + option (google.longrunning.operation_info) = { + response_type: "Project" + metadata_type: "UpdateProjectMetadata" + }; + } + + // Move a project to another place in your resource hierarchy, under a new + // resource parent. + // + // Returns an operation which can be used to track the process of the project + // move workflow. + // Upon success, the `Operation.response` field will be populated with the + // moved project. + // + // The caller must have `resourcemanager.projects.move` permission on the + // project, on the project's current and proposed new parent. + // + // If project has no current parent, or it currently does not have an + // associated organization resource, you will also need the + // `resourcemanager.projects.setIamPolicy` permission in the project. + // + // + rpc MoveProject(MoveProjectRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v3/{name=projects/*}:move" + body: "*" + }; + option (google.api.method_signature) = "name, destination_parent"; + option (google.longrunning.operation_info) = { + response_type: "Project" + metadata_type: "MoveProjectMetadata" + }; + } + + // Marks the project identified by the specified + // `name` (for example, `projects/415104041262`) for deletion. + // + // This method will only affect the project if it has a lifecycle state of + // [ACTIVE][google.cloud.resourcemanager.v3.Project.State.ACTIVE]. + // + // This method changes the Project's lifecycle state from + // [ACTIVE][google.cloud.resourcemanager.v3.Project.State.ACTIVE] + // to + // [DELETE_REQUESTED][google.cloud.resourcemanager.v3.Project.State.DELETE_REQUESTED]. + // The deletion starts at an unspecified time, + // at which point the Project is no longer accessible. + // + // Until the deletion completes, you can check the lifecycle state + // checked by retrieving the project with [GetProject] + // [google.cloud.resourcemanager.v3.Projects.GetProject], + // and the project remains visible to [ListProjects] + // [google.cloud.resourcemanager.v3.Projects.ListProjects]. + // However, you cannot update the project. + // + // After the deletion completes, the project is not retrievable by + // the [GetProject] + // [google.cloud.resourcemanager.v3.Projects.GetProject], + // [ListProjects] + // [google.cloud.resourcemanager.v3.Projects.ListProjects], and + // [SearchProjects][google.cloud.resourcemanager.v3.Projects.SearchProjects] + // methods. + // + // This method behaves idempotently, such that deleting a `DELETE_REQUESTED` + // project will not cause an error, but also won't do anything. + // + // The caller must have `resourcemanager.projects.delete` permissions for this + // project. + rpc DeleteProject(DeleteProjectRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v3/{name=projects/*}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "Project" + metadata_type: "DeleteProjectMetadata" + }; + } + + // Restores the project identified by the specified + // `name` (for example, `projects/415104041262`). + // You can only use this method for a project that has a lifecycle state of + // [DELETE_REQUESTED] + // [Projects.State.DELETE_REQUESTED]. + // After deletion starts, the project cannot be restored. + // + // The caller must have `resourcemanager.projects.undelete` permission for + // this project. + rpc UndeleteProject(UndeleteProjectRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v3/{name=projects/*}:undelete" + body: "*" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "Project" + metadata_type: "UndeleteProjectMetadata" + }; + } + + // Returns the IAM access control policy for the specified project, in the + // format `projects/{ProjectIdOrNumber}` e.g. projects/123. + // Permission is denied if the policy or the resource do not exist. + rpc GetIamPolicy(google.iam.v1.GetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + post: "/v3/{resource=projects/*}:getIamPolicy" + body: "*" + }; + option (google.api.method_signature) = "resource"; + } + + // Sets the IAM access control policy for the specified project, in the + // format `projects/{ProjectIdOrNumber}` e.g. projects/123. + // + // CAUTION: This method will replace the existing policy, and cannot be used + // to append additional IAM settings. + // + // Note: Removing service accounts from policies or changing their roles can + // render services completely inoperable. It is important to understand how + // the service account is being used before removing or updating its roles. + // + // The following constraints apply when using `setIamPolicy()`: + // + // + Project does not support `allUsers` and `allAuthenticatedUsers` as + // `members` in a `Binding` of a `Policy`. + // + // + The owner role can be granted to a `user`, `serviceAccount`, or a group + // that is part of an organization. For example, + // group@myownpersonaldomain.com could be added as an owner to a project in + // the myownpersonaldomain.com organization, but not the examplepetstore.com + // organization. + // + // + Service accounts can be made owners of a project directly + // without any restrictions. However, to be added as an owner, a user must be + // invited using the Cloud Platform console and must accept the invitation. + // + // + A user cannot be granted the owner role using `setIamPolicy()`. The user + // must be granted the owner role using the Cloud Platform Console and must + // explicitly accept the invitation. + // + // + Invitations to grant the owner role cannot be sent using + // `setIamPolicy()`; + // they must be sent only using the Cloud Platform Console. + // + // + If the project is not part of an organization, there must be at least + // one owner who has accepted the Terms of Service (ToS) agreement in the + // policy. Calling `setIamPolicy()` to remove the last ToS-accepted owner + // from the policy will fail. This restriction also applies to legacy + // projects that no longer have owners who have accepted the ToS. Edits to + // IAM policies will be rejected until the lack of a ToS-accepting owner is + // rectified. If the project is part of an organization, you can remove all + // owners, potentially making the organization inaccessible. + rpc SetIamPolicy(google.iam.v1.SetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + post: "/v3/{resource=projects/*}:setIamPolicy" + body: "*" + }; + option (google.api.method_signature) = "resource, policy"; + } + + // Returns permissions that a caller has on the specified project, in the + // format `projects/{ProjectIdOrNumber}` e.g. projects/123.. + rpc TestIamPermissions(google.iam.v1.TestIamPermissionsRequest) + returns (google.iam.v1.TestIamPermissionsResponse) { + option (google.api.http) = { + post: "/v3/{resource=projects/*}:testIamPermissions" + body: "*" + }; + option (google.api.method_signature) = "resource, permissions"; + } +} + +// A project is a high-level Google Cloud entity. It is a +// container for ACLs, APIs, App Engine Apps, VMs, and other +// Google Cloud Platform resources. +message Project { + option (google.api.resource) = { + type: "cloudresourcemanager.googleapis.com/Project" + pattern: "projects/{project}" + style: DECLARATIVE_FRIENDLY + }; + + // Project lifecycle states. + enum State { + // Unspecified state. This is only used/useful for distinguishing + // unset values. + STATE_UNSPECIFIED = 0; + + // The normal and active state. + ACTIVE = 1; + + // The project has been marked for deletion by the user + // (by invoking + // [DeleteProject][google.cloud.resourcemanager.v3.Projects.DeleteProject]) + // or by the system (Google Cloud Platform). + // This can generally be reversed by invoking [UndeleteProject] + // [google.cloud.resourcemanager.v3.Projects.UndeleteProject]. + DELETE_REQUESTED = 2; + } + + // Output only. The unique resource name of the project. It is an int64 + // generated number prefixed by "projects/". + // + // Example: `projects/415104041262` + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. A reference to a parent Resource. eg., `organizations/123` or + // `folders/876`. + string parent = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Immutable. The unique, user-assigned id of the project. + // It must be 6 to 30 lowercase ASCII letters, digits, or hyphens. + // It must start with a letter. + // Trailing hyphens are prohibited. + // + // Example: `tokyo-rain-123` + string project_id = 3 [(google.api.field_behavior) = IMMUTABLE]; + + // Output only. The project lifecycle state. + State state = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. A user-assigned display name of the project. + // When present it must be between 4 to 30 characters. + // Allowed characters are: lowercase and uppercase letters, numbers, + // hyphen, single-quote, double-quote, space, and exclamation point. + // + // Example: `My Project` + string display_name = 5 [(google.api.field_behavior) = OPTIONAL]; + + // Output only. Creation time. + google.protobuf.Timestamp create_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The most recent time this resource was modified. + google.protobuf.Timestamp update_time = 7 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time at which this resource was requested for deletion. + google.protobuf.Timestamp delete_time = 8 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. A checksum computed by the server based on the current value + // of the Project resource. This may be sent on update and delete requests to + // ensure the client has an up-to-date value before proceeding. + string etag = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. The labels associated with this project. + // + // Label keys must be between 1 and 63 characters long and must conform + // to the following regular expression: \[a-z\](\[-a-z0-9\]*\[a-z0-9\])?. + // + // Label values must be between 0 and 63 characters long and must conform + // to the regular expression (\[a-z\](\[-a-z0-9\]*\[a-z0-9\])?)?. + // + // No more than 64 labels can be associated with a given resource. + // + // Clients should store labels in a representation such as JSON that does not + // depend on specific characters being disallowed. + // + // Example: `"myBusinessDimension" : "businessValue"` + map labels = 10 [(google.api.field_behavior) = OPTIONAL]; +} + +// The request sent to the +// [GetProject][google.cloud.resourcemanager.v3.Projects.GetProject] +// method. +message GetProjectRequest { + // Required. The name of the project (for example, `projects/415104041262`). + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; +} + +// The request sent to the +// [ListProjects][google.cloud.resourcemanager.v3.Projects.ListProjects] +// method. +message ListProjectsRequest { + // Required. The name of the parent resource whose projects are being listed. + // Only children of this parent resource are listed; descendants are not + // listed. + // + // If the parent is a folder, use the value `folders/{folder_id}`. If the + // parent is an organization, use the value `organizations/{org_id}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { child_type: "*" } + ]; + + // Optional. A pagination token returned from a previous call to + // [ListProjects] [google.cloud.resourcemanager.v3.Projects.ListProjects] that + // indicates from where listing should continue. + string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The maximum number of projects to return in the response. + // The server can return fewer projects than requested. + // If unspecified, server picks an appropriate default. + int32 page_size = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Indicate that projects in the `DELETE_REQUESTED` state should + // also be returned. Normally only `ACTIVE` projects are returned. + bool show_deleted = 4 [(google.api.field_behavior) = OPTIONAL]; +} + +// A page of the response received from the +// [ListProjects][google.cloud.resourcemanager.v3.Projects.ListProjects] +// method. +// +// A paginated response where more pages are available has +// `next_page_token` set. This token can be used in a subsequent request to +// retrieve the next request page. +// +// NOTE: A response may contain fewer elements than the request `page_size` and +// still have a `next_page_token`. +message ListProjectsResponse { + // The list of Projects under the parent. This list can be paginated. + repeated Project projects = 1; + + // Pagination token. + // + // If the result set is too large to fit in a single response, this token + // is returned. It encodes the position of the current result cursor. + // Feeding this value into a new list request with the `page_token` parameter + // gives the next page of the results. + // + // When `next_page_token` is not filled in, there is no next page and + // the list returned is the last page in the result set. + // + // Pagination tokens have a limited lifetime. + string next_page_token = 2; +} + +// The request sent to the +// [SearchProjects][google.cloud.resourcemanager.v3.Projects.SearchProjects] +// method. +message SearchProjectsRequest { + // Optional. A query string for searching for projects that the caller has + // `resourcemanager.projects.get` permission to. If multiple fields are + // included in the query, then it will return results that match any of the + // fields. Some eligible fields are: + // + // - **`displayName`, `name`**: Filters by displayName. + // - **`parent`**: Project's parent (for example: `folders/123`, + // `organizations/*`). Prefer `parent` field over `parent.type` and + // `parent.id`. + // - **`parent.type`**: Parent's type: `folder` or `organization`. + // - **`parent.id`**: Parent's id number (for example: `123`). + // - **`id`, `projectId`**: Filters by projectId. + // - **`state`, `lifecycleState`**: Filters by state. + // - **`labels`**: Filters by label name or value. + // - **`labels.` (where `` is the name of a label)**: Filters by label + // name. + // + // Search expressions are case insensitive. + // + // Some examples queries: + // + // + // - **`name:how*`**: The project's name starts with "how". + // - **`name:Howl`**: The project's name is `Howl` or `howl`. + // - **`name:HOWL`**: Equivalent to above. + // - **`NAME:howl`**: Equivalent to above. + // - **`labels.color:*`**: The project has the label `color`. + // - **`labels.color:red`**: The project's label `color` has the value `red`. + // - **`labels.color:red labels.size:big`**: The project's label `color` has + // the value `red` or its label `size` has the value `big`. + // + // If no query is specified, the call will return projects for which the user + // has the `resourcemanager.projects.get` permission. + string query = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to + // [ListProjects] [google.cloud.resourcemanager.v3.Projects.ListProjects] that + // indicates from where listing should continue. + string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The maximum number of projects to return in the response. + // The server can return fewer projects than requested. + // If unspecified, server picks an appropriate default. + int32 page_size = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// A page of the response received from the +// [SearchProjects][google.cloud.resourcemanager.v3.Projects.SearchProjects] +// method. +// +// A paginated response where more pages are available has +// `next_page_token` set. This token can be used in a subsequent request to +// retrieve the next request page. +message SearchProjectsResponse { + // The list of Projects that matched the list filter query. This list can + // be paginated. + repeated Project projects = 1; + + // Pagination token. + // + // If the result set is too large to fit in a single response, this token + // is returned. It encodes the position of the current result cursor. + // Feeding this value into a new list request with the `page_token` parameter + // gives the next page of the results. + // + // When `next_page_token` is not filled in, there is no next page and + // the list returned is the last page in the result set. + // + // Pagination tokens have a limited lifetime. + string next_page_token = 2; +} + +// The request sent to the +// [CreateProject][google.cloud.resourcemanager.v3.Projects.CreateProject] +// method. +message CreateProjectRequest { + // Required. The Project to create. + // + // Project ID is required. If the requested ID is unavailable, the request + // fails. + // + // If the `parent` field is set, the `resourcemanager.projects.create` + // permission is checked on the parent resource. If no parent is set and + // the authorization credentials belong to an Organization, the parent + // will be set to that Organization. + Project project = 1 [(google.api.field_behavior) = REQUIRED]; +} + +// A status object which is used as the `metadata` field for the Operation +// returned by CreateProject. It provides insight for when significant phases of +// Project creation have completed. +message CreateProjectMetadata { + // Creation time of the project creation workflow. + google.protobuf.Timestamp create_time = 1; + + // True if the project can be retrieved using `GetProject`. No other + // operations on the project are guaranteed to work until the project creation + // is complete. + bool gettable = 2; + + // True if the project creation process is complete. + bool ready = 3; +} + +// The request sent to the +// [UpdateProject][google.cloud.resourcemanager.v3.Projects.UpdateProject] +// method. +// +// Only the `display_name` and `labels` fields can be change. Use the +// [MoveProject][google.cloud.resourcemanager.v3.Projects.MoveProject] method to +// change the `parent` field. +message UpdateProjectRequest { + // Required. The new definition of the project. + Project project = 1 [(google.api.field_behavior) = REQUIRED]; + + // Optional. An update mask to selectively update fields. + google.protobuf.FieldMask update_mask = 2 + [(google.api.field_behavior) = OPTIONAL]; +} + +// A status object which is used as the `metadata` field for the Operation +// returned by UpdateProject. +message UpdateProjectMetadata {} + +// The request sent to +// [MoveProject][google.cloud.resourcemanager.v3.Projects.MoveProject] +// method. +message MoveProjectRequest { + // Required. The name of the project to move. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; + + // Required. The new parent to move the Project under. + string destination_parent = 2 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { child_type: "*" } + ]; +} + +// A status object which is used as the `metadata` field for the Operation +// returned by MoveProject. +message MoveProjectMetadata {} + +// [DeleteProject][google.cloud.resourcemanager.v3.Projects.DeleteProject] +// method. +message DeleteProjectRequest { + // Required. The name of the Project (for example, `projects/415104041262`). + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; +} + +// A status object which is used as the `metadata` field for the Operation +// returned by `DeleteProject`. +message DeleteProjectMetadata {} + +// The request sent to the [UndeleteProject] +// [google.cloud.resourcemanager.v3.Projects.UndeleteProject] +// method. +message UndeleteProjectRequest { + // Required. The name of the project (for example, `projects/415104041262`). + // + // Required. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; +} + +// A status object which is used as the `metadata` field for the Operation +// returned by `UndeleteProject`. +message UndeleteProjectMetadata {} diff --git a/proto/google/cloud/resourcemanager/v3/tag_bindings.proto b/proto/google/cloud/resourcemanager/v3/tag_bindings.proto new file mode 100644 index 0000000..dcdb6b5 --- /dev/null +++ b/proto/google/cloud/resourcemanager/v3/tag_bindings.proto @@ -0,0 +1,268 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.resourcemanager.v3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/empty.proto"; + +option csharp_namespace = "Google.Cloud.ResourceManager.V3"; +option go_package = "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb;resourcemanagerpb"; +option java_multiple_files = true; +option java_outer_classname = "TagBindingsProto"; +option java_package = "com.google.cloud.resourcemanager.v3"; +option php_namespace = "Google\\Cloud\\ResourceManager\\V3"; +option ruby_package = "Google::Cloud::ResourceManager::V3"; + +// Allow users to create and manage TagBindings between TagValues and +// different Google Cloud resources throughout the GCP resource hierarchy. +service TagBindings { + option (google.api.default_host) = "cloudresourcemanager.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/cloud-platform.read-only"; + + // Lists the TagBindings for the given Google Cloud resource, as specified + // with `parent`. + // + // NOTE: The `parent` field is expected to be a full resource name: + // https://cloud.google.com/apis/design/resource_names#full_resource_name + rpc ListTagBindings(ListTagBindingsRequest) + returns (ListTagBindingsResponse) { + option (google.api.http) = { + get: "/v3/tagBindings" + }; + option (google.api.method_signature) = "parent"; + } + + // Creates a TagBinding between a TagValue and a Google Cloud resource. + rpc CreateTagBinding(CreateTagBindingRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v3/tagBindings" + body: "tag_binding" + }; + option (google.api.method_signature) = "tag_binding"; + option (google.longrunning.operation_info) = { + response_type: "TagBinding" + metadata_type: "CreateTagBindingMetadata" + }; + } + + // Deletes a TagBinding. + rpc DeleteTagBinding(DeleteTagBindingRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v3/{name=tagBindings/**}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "DeleteTagBindingMetadata" + }; + } + + // Return a list of effective tags for the given Google Cloud resource, as + // specified in `parent`. + rpc ListEffectiveTags(ListEffectiveTagsRequest) + returns (ListEffectiveTagsResponse) { + option (google.api.http) = { + get: "/v3/effectiveTags" + }; + option (google.api.method_signature) = "parent"; + } +} + +// A TagBinding represents a connection between a TagValue and a cloud +// resource Once a TagBinding is created, the TagValue is applied to all the +// descendants of the Google Cloud resource. +message TagBinding { + option (google.api.resource) = { + type: "cloudresourcemanager.googleapis.com/TagBinding" + pattern: "tagBindings/{tag_binding}" + }; + + // Output only. The name of the TagBinding. This is a String of the form: + // `tagBindings/{full-resource-name}/{tag-value-name}` (e.g. + // `tagBindings/%2F%2Fcloudresourcemanager.googleapis.com%2Fprojects%2F123/tagValues/456`). + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // The full resource name of the resource the TagValue is bound to. + // E.g. `//cloudresourcemanager.googleapis.com/projects/123` + string parent = 2; + + // The TagValue of the TagBinding. + // Must be of the form `tagValues/456`. + string tag_value = 3; + + // The namespaced name for the TagValue of the TagBinding. + // Must be in the format + // `{parent_id}/{tag_key_short_name}/{short_name}`. + // + // For methods that support TagValue namespaced name, only one of + // tag_value_namespaced_name or tag_value may be filled. Requests with both + // fields will be rejected. + string tag_value_namespaced_name = 4; +} + +// Runtime operation information for creating a TagValue. +message CreateTagBindingMetadata {} + +// The request message to create a TagBinding. +message CreateTagBindingRequest { + // Required. The TagBinding to be created. + TagBinding tag_binding = 1 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Set to true to perform the validations necessary for creating the + // resource, but not actually perform the action. + bool validate_only = 2 [(google.api.field_behavior) = OPTIONAL]; +} + +// Runtime operation information for deleting a TagBinding. +message DeleteTagBindingMetadata {} + +// The request message to delete a TagBinding. +message DeleteTagBindingRequest { + // Required. The name of the TagBinding. This is a String of the form: + // `tagBindings/{id}` (e.g. + // `tagBindings/%2F%2Fcloudresourcemanager.googleapis.com%2Fprojects%2F123/tagValues/456`). + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/TagBinding" + } + ]; +} + +// The request message to list all TagBindings for a parent. +message ListTagBindingsRequest { + // Required. The full resource name of a resource for which you want to list + // existing TagBindings. E.g. + // "//cloudresourcemanager.googleapis.com/projects/123" + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { child_type: "*" } + ]; + + // Optional. The maximum number of TagBindings to return in the response. The + // server allows a maximum of 300 TagBindings to return. If unspecified, the + // server will use 100 as the default. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to + // `ListTagBindings` that indicates where this listing should continue from. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// The ListTagBindings response. +message ListTagBindingsResponse { + // A possibly paginated list of TagBindings for the specified resource. + repeated TagBinding tag_bindings = 1; + + // Pagination token. + // + // If the result set is too large to fit in a single response, this token + // is returned. It encodes the position of the current result cursor. + // Feeding this value into a new list request with the `page_token` parameter + // gives the next page of the results. + // + // When `next_page_token` is not filled in, there is no next page and + // the list returned is the last page in the result set. + // + // Pagination tokens have a limited lifetime. + string next_page_token = 2; +} + +// The request message to ListEffectiveTags +message ListEffectiveTagsRequest { + // Required. The full resource name of a resource for which you want to list + // the effective tags. E.g. + // "//cloudresourcemanager.googleapis.com/projects/123" + string parent = 1 [(google.api.field_behavior) = REQUIRED]; + + // Optional. The maximum number of effective tags to return in the response. + // The server allows a maximum of 300 effective tags to return in a single + // page. If unspecified, the server will use 100 as the default. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to + // `ListEffectiveTags` that indicates from where this listing should continue. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// The response of ListEffectiveTags. +message ListEffectiveTagsResponse { + // A possibly paginated list of effective tags for the specified resource. + repeated EffectiveTag effective_tags = 1; + + // Pagination token. + // + // If the result set is too large to fit in a single response, this token + // is returned. It encodes the position of the current result cursor. + // Feeding this value into a new list request with the `page_token` parameter + // gives the next page of the results. + // + // When `next_page_token` is not filled in, there is no next page and + // the list returned is the last page in the result set. + // + // Pagination tokens have a limited lifetime. + string next_page_token = 2; +} + +// An EffectiveTag represents a tag that applies to a resource during policy +// evaluation. Tags can be either directly bound to a resource or inherited from +// its ancestor. EffectiveTag contains the name and +// namespaced_name of the tag value and tag key, with additional fields of +// `inherited` to indicate the inheritance status of the effective tag. +message EffectiveTag { + // Resource name for TagValue in the format `tagValues/456`. + string tag_value = 1 [(google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/TagValue" + }]; + + // The namespaced name of the TagValue. Can be in the form + // `{organization_id}/{tag_key_short_name}/{tag_value_short_name}` or + // `{project_id}/{tag_key_short_name}/{tag_value_short_name}` or + // `{project_number}/{tag_key_short_name}/{tag_value_short_name}`. + string namespaced_tag_value = 2; + + // The name of the TagKey, in the format `tagKeys/{id}`, such as + // `tagKeys/123`. + string tag_key = 3 [(google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/TagKey" + }]; + + // The namespaced name of the TagKey. Can be in the form + // `{organization_id}/{tag_key_short_name}` or + // `{project_id}/{tag_key_short_name}` or + // `{project_number}/{tag_key_short_name}`. + string namespaced_tag_key = 4; + + // The parent name of the tag key. + // Must be in the format `organizations/{organization_id}` or + // `projects/{project_number}` + string tag_key_parent_name = 6; + + // Indicates the inheritance status of a tag value + // attached to the given resource. If the tag value is inherited from one of + // the resource's ancestors, inherited will be true. If false, then the tag + // value is directly attached to the resource, inherited will be false. + bool inherited = 5; +} diff --git a/proto/google/cloud/resourcemanager/v3/tag_holds.proto b/proto/google/cloud/resourcemanager/v3/tag_holds.proto new file mode 100644 index 0000000..0605ad7 --- /dev/null +++ b/proto/google/cloud/resourcemanager/v3/tag_holds.proto @@ -0,0 +1,216 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.resourcemanager.v3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ResourceManager.V3"; +option go_package = "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb;resourcemanagerpb"; +option java_multiple_files = true; +option java_outer_classname = "TagHoldsProto"; +option java_package = "com.google.cloud.resourcemanager.v3"; +option php_namespace = "Google\\Cloud\\ResourceManager\\V3"; +option ruby_package = "Google::Cloud::ResourceManager::V3"; + +// Allow users to create and manage TagHolds for TagValues. TagHolds represent +// the use of a Tag Value that is not captured by TagBindings but +// should still block TagValue deletion (such as a reference in a policy +// condition). This service provides isolated failure domains by cloud location +// so that TagHolds can be managed in the same location as their usage. +service TagHolds { + option (google.api.default_host) = "cloudresourcemanager.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/cloud-platform.read-only"; + + // Creates a TagHold. Returns ALREADY_EXISTS if a TagHold with the same + // resource and origin exists under the same TagValue. + rpc CreateTagHold(CreateTagHoldRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v3/{parent=tagValues/*}/tagHolds" + body: "tag_hold" + }; + option (google.api.method_signature) = "parent,tag_hold"; + option (google.longrunning.operation_info) = { + response_type: "TagHold" + metadata_type: "CreateTagHoldMetadata" + }; + } + + // Deletes a TagHold. + rpc DeleteTagHold(DeleteTagHoldRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v3/{name=tagValues/*/tagHolds/*}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "DeleteTagHoldMetadata" + }; + } + + // Lists TagHolds under a TagValue. + rpc ListTagHolds(ListTagHoldsRequest) returns (ListTagHoldsResponse) { + option (google.api.http) = { + get: "/v3/{parent=tagValues/*}/tagHolds" + }; + option (google.api.method_signature) = "parent"; + } +} + +// A TagHold represents the use of a TagValue that is not captured by +// TagBindings. If a TagValue has any TagHolds, deletion will be blocked. +// This resource is intended to be created in the same cloud location as the +// `holder`. +message TagHold { + option (google.api.resource) = { + type: "cloudresourcemanager.googleapis.com/TagHold" + pattern: "tagValues/{tag_value}/tagHolds/{tag_hold}" + }; + + // Output only. The resource name of a TagHold. This is a String of the form: + // `tagValues/{tag-value-id}/tagHolds/{tag-hold-id}` + // (e.g. `tagValues/123/tagHolds/456`). This resource name is generated by + // the server. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Required. The name of the resource where the TagValue is being used. Must + // be less than 200 characters. E.g. + // `//compute.googleapis.com/compute/projects/myproject/regions/us-east-1/instanceGroupManagers/instance-group` + string holder = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. An optional string representing the origin of this request. This + // field should include human-understandable information to distinguish + // origins from each other. Must be less than 200 characters. E.g. + // `migs-35678234` + string origin = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A URL where an end user can learn more about removing this hold. + // E.g. + // `https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing` + string help_link = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Output only. The time this TagHold was created. + google.protobuf.Timestamp create_time = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// The request message to create a TagHold. +message CreateTagHoldRequest { + // Required. The resource name of the TagHold's parent TagValue. Must be of + // the form: `tagValues/{tag-value-id}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "cloudresourcemanager.googleapis.com/TagHold" + } + ]; + + // Required. The TagHold to be created. + TagHold tag_hold = 2 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Set to true to perform the validations necessary for creating the + // resource, but not actually perform the action. + bool validate_only = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// Runtime operation information for creating a TagHold. +// (-- The metadata is currently empty, but may include information in the +// future. --) +message CreateTagHoldMetadata {} + +// The request message to delete a TagHold. +message DeleteTagHoldRequest { + // Required. The resource name of the TagHold to delete. Must be of the form: + // `tagValues/{tag-value-id}/tagHolds/{tag-hold-id}`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/TagHold" + } + ]; + + // Optional. Set to true to perform the validations necessary for deleting the + // resource, but not actually perform the action. + bool validate_only = 2 [(google.api.field_behavior) = OPTIONAL]; +} + +// Runtime operation information for deleting a TagHold. +// (-- The metadata is currently empty, but may include information in the +// future. --) +message DeleteTagHoldMetadata {} + +// The request message for listing the TagHolds under a TagValue. +message ListTagHoldsRequest { + // Required. The resource name of the parent TagValue. Must be of the form: + // `tagValues/{tag-value-id}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "cloudresourcemanager.googleapis.com/TagHold" + } + ]; + + // Optional. The maximum number of TagHolds to return in the response. The + // server allows a maximum of 300 TagHolds to return. If unspecified, the + // server will use 100 as the default. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to + // `ListTagHolds` that indicates where this listing should continue from. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Criteria used to select a subset of TagHolds parented by the + // TagValue to return. This field follows the syntax defined by aip.dev/160; + // the `holder` and `origin` fields are supported for filtering. Currently + // only `AND` syntax is supported. Some example queries are: + // + // * `holder = + // //compute.googleapis.com/compute/projects/myproject/regions/us-east-1/instanceGroupManagers/instance-group` + // * `origin = 35678234` + // * `holder = + // //compute.googleapis.com/compute/projects/myproject/regions/us-east-1/instanceGroupManagers/instance-group + // AND origin = 35678234` + string filter = 4 [(google.api.field_behavior) = OPTIONAL]; +} + +// The ListTagHolds response. +message ListTagHoldsResponse { + // A possibly paginated list of TagHolds. + repeated TagHold tag_holds = 1; + + // Pagination token. + // + // If the result set is too large to fit in a single response, this token + // is returned. It encodes the position of the current result cursor. + // Feeding this value into a new list request with the `page_token` parameter + // gives the next page of the results. + // + // When `next_page_token` is not filled in, there is no next page and + // the list returned is the last page in the result set. + // + // Pagination tokens have a limited lifetime. + string next_page_token = 2; +} diff --git a/proto/google/cloud/resourcemanager/v3/tag_keys.proto b/proto/google/cloud/resourcemanager/v3/tag_keys.proto new file mode 100644 index 0000000..5dbe94b --- /dev/null +++ b/proto/google/cloud/resourcemanager/v3/tag_keys.proto @@ -0,0 +1,367 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.resourcemanager.v3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/iam/v1/iam_policy.proto"; +import "google/iam/v1/policy.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ResourceManager.V3"; +option go_package = "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb;resourcemanagerpb"; +option java_multiple_files = true; +option java_outer_classname = "TagKeysProto"; +option java_package = "com.google.cloud.resourcemanager.v3"; +option php_namespace = "Google\\Cloud\\ResourceManager\\V3"; +option ruby_package = "Google::Cloud::ResourceManager::V3"; + +// Allow users to create and manage tag keys. +service TagKeys { + option (google.api.default_host) = "cloudresourcemanager.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/cloud-platform.read-only"; + + // Lists all TagKeys for a parent resource. + rpc ListTagKeys(ListTagKeysRequest) returns (ListTagKeysResponse) { + option (google.api.http) = { + get: "/v3/tagKeys" + }; + option (google.api.method_signature) = "parent"; + } + + // Retrieves a TagKey. This method will return `PERMISSION_DENIED` if the + // key does not exist or the user does not have permission to view it. + rpc GetTagKey(GetTagKeyRequest) returns (TagKey) { + option (google.api.http) = { + get: "/v3/{name=tagKeys/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Retrieves a TagKey by its namespaced name. + // This method will return `PERMISSION_DENIED` if the key does not exist + // or the user does not have permission to view it. + rpc GetNamespacedTagKey(GetNamespacedTagKeyRequest) returns (TagKey) { + option (google.api.http) = { + get: "/v3/tagKeys/namespaced" + }; + option (google.api.method_signature) = "name"; + } + + // Creates a new TagKey. If another request with the same parameters is + // sent while the original request is in process, the second request + // will receive an error. A maximum of 1000 TagKeys can exist under a parent + // at any given time. + rpc CreateTagKey(CreateTagKeyRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v3/tagKeys" + body: "tag_key" + }; + option (google.api.method_signature) = "tag_key"; + option (google.longrunning.operation_info) = { + response_type: "TagKey" + metadata_type: "CreateTagKeyMetadata" + }; + } + + // Updates the attributes of the TagKey resource. + rpc UpdateTagKey(UpdateTagKeyRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { + patch: "/v3/{tag_key.name=tagKeys/*}" + body: "tag_key" + }; + option (google.api.method_signature) = "tag_key,update_mask"; + option (google.longrunning.operation_info) = { + response_type: "TagKey" + metadata_type: "UpdateTagKeyMetadata" + }; + } + + // Deletes a TagKey. The TagKey cannot be deleted if it has any child + // TagValues. + rpc DeleteTagKey(DeleteTagKeyRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v3/{name=tagKeys/*}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "TagKey" + metadata_type: "DeleteTagKeyMetadata" + }; + } + + // Gets the access control policy for a TagKey. The returned policy may be + // empty if no such policy or resource exists. The `resource` field should + // be the TagKey's resource name. For example, "tagKeys/1234". + // The caller must have + // `cloudresourcemanager.googleapis.com/tagKeys.getIamPolicy` permission on + // the specified TagKey. + rpc GetIamPolicy(google.iam.v1.GetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + post: "/v3/{resource=tagKeys/*}:getIamPolicy" + body: "*" + }; + option (google.api.method_signature) = "resource"; + } + + // Sets the access control policy on a TagKey, replacing any existing + // policy. The `resource` field should be the TagKey's resource name. + // For example, "tagKeys/1234". + // The caller must have `resourcemanager.tagKeys.setIamPolicy` permission + // on the identified tagValue. + rpc SetIamPolicy(google.iam.v1.SetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + post: "/v3/{resource=tagKeys/*}:setIamPolicy" + body: "*" + }; + option (google.api.method_signature) = "resource,policy"; + } + + // Returns permissions that a caller has on the specified TagKey. + // The `resource` field should be the TagKey's resource name. + // For example, "tagKeys/1234". + // + // There are no permissions required for making this API call. + rpc TestIamPermissions(google.iam.v1.TestIamPermissionsRequest) + returns (google.iam.v1.TestIamPermissionsResponse) { + option (google.api.http) = { + post: "/v3/{resource=tagKeys/*}:testIamPermissions" + body: "*" + }; + option (google.api.method_signature) = "resource,permissions"; + } +} + +// A TagKey, used to group a set of TagValues. +message TagKey { + option (google.api.resource) = { + type: "cloudresourcemanager.googleapis.com/TagKey" + pattern: "tagKeys/{tag_key}" + style: DECLARATIVE_FRIENDLY + }; + + // Immutable. The resource name for a TagKey. Must be in the format + // `tagKeys/{tag_key_id}`, where `tag_key_id` is the generated numeric id for + // the TagKey. + string name = 1 [(google.api.field_behavior) = IMMUTABLE]; + + // Immutable. The resource name of the TagKey's parent. A TagKey can be + // parented by an Organization or a Project. For a TagKey parented by an + // Organization, its parent must be in the form `organizations/{org_id}`. For + // a TagKey parented by a Project, its parent can be in the form + // `projects/{project_id}` or `projects/{project_number}`. + string parent = 2 [(google.api.field_behavior) = IMMUTABLE]; + + // Required. Immutable. The user friendly name for a TagKey. The short name + // should be unique for TagKeys within the same tag namespace. + // + // The short name must be 1-63 characters, beginning and ending with + // an alphanumeric character ([a-z0-9A-Z]) with dashes (-), underscores (_), + // dots (.), and alphanumerics between. + string short_name = 3 [ + (google.api.field_behavior) = REQUIRED, + (google.api.field_behavior) = IMMUTABLE + ]; + + // Output only. Immutable. Namespaced name of the TagKey. + string namespaced_name = 4 [ + (google.api.field_behavior) = OUTPUT_ONLY, + (google.api.field_behavior) = IMMUTABLE + ]; + + // Optional. User-assigned description of the TagKey. Must not exceed 256 + // characters. + // + // Read-write. + string description = 5 [(google.api.field_behavior) = OPTIONAL]; + + // Output only. Creation time. + google.protobuf.Timestamp create_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Update time. + google.protobuf.Timestamp update_time = 7 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. Entity tag which users can pass to prevent race conditions. This + // field is always set in server responses. See UpdateTagKeyRequest for + // details. + string etag = 8 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A purpose denotes that this Tag is intended for use in policies + // of a specific policy engine, and will involve that policy engine in + // management operations involving this Tag. A purpose does not grant a + // policy engine exclusive rights to the Tag, and it may be referenced by + // other policy engines. + // + // A purpose cannot be changed once set. + Purpose purpose = 11 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Purpose data corresponds to the policy system that the tag is + // intended for. See documentation for `Purpose` for formatting of this field. + // + // Purpose data cannot be changed once set. + map purpose_data = 12 + [(google.api.field_behavior) = OPTIONAL]; +} + +// The request message for listing all TagKeys under a parent resource. +message ListTagKeysRequest { + // Required. The resource name of the TagKey's parent. + // Must be of the form `organizations/{org_id}` or `projects/{project_id}` or + // `projects/{project_number}` + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { child_type: "*" } + ]; + + // Optional. The maximum number of TagKeys to return in the response. The + // server allows a maximum of 300 TagKeys to return. If unspecified, the + // server will use 100 as the default. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to `ListTagKey` + // that indicates where this listing should continue from. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// The ListTagKeys response message. +message ListTagKeysResponse { + // List of TagKeys that live under the specified parent in the request. + repeated TagKey tag_keys = 1; + + // A pagination token returned from a previous call to `ListTagKeys` + // that indicates from where listing should continue. + string next_page_token = 2; +} + +// The request message for getting a TagKey. +message GetTagKeyRequest { + // Required. A resource name in the format `tagKeys/{id}`, such as + // `tagKeys/123`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/TagKey" + } + ]; +} + +// The request message for getting a TagKey by its namespaced name. +message GetNamespacedTagKeyRequest { + // Required. A namespaced tag key name in the format + // `{parentId}/{tagKeyShort}`, such as `42/foo` for a key with short name + // "foo" under the organization with ID 42 or `r2-d2/bar` for a key with short + // name "bar" under the project `r2-d2`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/TagKey" + } + ]; +} + +// The request message for creating a TagKey. +message CreateTagKeyRequest { + // Required. The TagKey to be created. Only fields `short_name`, + // `description`, and `parent` are considered during the creation request. + TagKey tag_key = 1 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Set to true to perform validations necessary for creating the + // resource, but not actually perform the action. + bool validate_only = 2 [(google.api.field_behavior) = OPTIONAL]; +} + +// Runtime operation information for creating a TagKey. +message CreateTagKeyMetadata {} + +// The request message for updating a TagKey. +message UpdateTagKeyRequest { + // Required. The new definition of the TagKey. Only the `description` and + // `etag` fields can be updated by this request. If the `etag` field is not + // empty, it must match the `etag` field of the existing tag key. Otherwise, + // `ABORTED` will be returned. + TagKey tag_key = 1 [(google.api.field_behavior) = REQUIRED]; + + // Fields to be updated. The mask may only contain `description` or + // `etag`. If omitted entirely, both `description` and `etag` are assumed to + // be significant. + google.protobuf.FieldMask update_mask = 2; + + // Set as true to perform validations necessary for updating the resource, but + // not actually perform the action. + bool validate_only = 3; +} + +// Runtime operation information for updating a TagKey. +message UpdateTagKeyMetadata {} + +// The request message for deleting a TagKey. +message DeleteTagKeyRequest { + // Required. The resource name of a TagKey to be deleted in the format + // `tagKeys/123`. The TagKey cannot be a parent of any existing TagValues or + // it will not be deleted successfully. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/TagKey" + } + ]; + + // Optional. Set as true to perform validations necessary for deletion, but + // not actually perform the action. + bool validate_only = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The etag known to the client for the expected state of the + // TagKey. This is to be used for optimistic concurrency. + string etag = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// Runtime operation information for deleting a TagKey. +message DeleteTagKeyMetadata {} + +// A purpose for each policy engine requiring such an integration. A single +// policy engine may have multiple purposes defined, however a TagKey may only +// specify a single purpose. +enum Purpose { + // Unspecified purpose. + PURPOSE_UNSPECIFIED = 0; + + // Purpose for Compute Engine firewalls. + // A corresponding `purpose_data` should be set for the network the tag is + // intended for. The key should be `network` and the value should be in + // either of these two formats: + // + // - + // `https://www.googleapis.com/compute/{compute_version}/projects/{project_id}/global/networks/{network_id}` + // - `{project_id}/{network_name}` + // + // Examples: + // + // - + // `https://www.googleapis.com/compute/staging_v1/projects/fail-closed-load-testing/global/networks/6992953698831725600` + // - `fail-closed-load-testing/load-testing-network` + GCE_FIREWALL = 1; +} diff --git a/proto/google/cloud/resourcemanager/v3/tag_values.proto b/proto/google/cloud/resourcemanager/v3/tag_values.proto new file mode 100644 index 0000000..dc7f5a9 --- /dev/null +++ b/proto/google/cloud/resourcemanager/v3/tag_values.proto @@ -0,0 +1,331 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.resourcemanager.v3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/iam/v1/iam_policy.proto"; +import "google/iam/v1/policy.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ResourceManager.V3"; +option go_package = "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb;resourcemanagerpb"; +option java_multiple_files = true; +option java_outer_classname = "TagValuesProto"; +option java_package = "com.google.cloud.resourcemanager.v3"; +option php_namespace = "Google\\Cloud\\ResourceManager\\V3"; +option ruby_package = "Google::Cloud::ResourceManager::V3"; + +// Allow users to create and manage tag values. +service TagValues { + option (google.api.default_host) = "cloudresourcemanager.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/cloud-platform.read-only"; + + // Lists all TagValues for a specific TagKey. + rpc ListTagValues(ListTagValuesRequest) returns (ListTagValuesResponse) { + option (google.api.http) = { + get: "/v3/tagValues" + }; + option (google.api.method_signature) = "parent"; + } + + // Retrieves a TagValue. This method will return `PERMISSION_DENIED` if the + // value does not exist or the user does not have permission to view it. + rpc GetTagValue(GetTagValueRequest) returns (TagValue) { + option (google.api.http) = { + get: "/v3/{name=tagValues/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Retrieves a TagValue by its namespaced name. + // This method will return `PERMISSION_DENIED` if the value does not exist + // or the user does not have permission to view it. + rpc GetNamespacedTagValue(GetNamespacedTagValueRequest) returns (TagValue) { + option (google.api.http) = { + get: "/v3/tagValues/namespaced" + }; + option (google.api.method_signature) = "name"; + } + + // Creates a TagValue as a child of the specified TagKey. If a another + // request with the same parameters is sent while the original request is in + // process the second request will receive an error. A maximum of 1000 + // TagValues can exist under a TagKey at any given time. + rpc CreateTagValue(CreateTagValueRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v3/tagValues" + body: "tag_value" + }; + option (google.api.method_signature) = "tag_value"; + option (google.longrunning.operation_info) = { + response_type: "TagValue" + metadata_type: "CreateTagValueMetadata" + }; + } + + // Updates the attributes of the TagValue resource. + rpc UpdateTagValue(UpdateTagValueRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + patch: "/v3/{tag_value.name=tagValues/*}" + body: "tag_value" + }; + option (google.api.method_signature) = "tag_value,update_mask"; + option (google.longrunning.operation_info) = { + response_type: "TagValue" + metadata_type: "UpdateTagValueMetadata" + }; + } + + // Deletes a TagValue. The TagValue cannot have any bindings when it is + // deleted. + rpc DeleteTagValue(DeleteTagValueRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v3/{name=tagValues/*}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "TagValue" + metadata_type: "DeleteTagValueMetadata" + }; + } + + // Gets the access control policy for a TagValue. The returned policy may be + // empty if no such policy or resource exists. The `resource` field should + // be the TagValue's resource name. For example: `tagValues/1234`. + // The caller must have the + // `cloudresourcemanager.googleapis.com/tagValues.getIamPolicy` permission on + // the identified TagValue to get the access control policy. + rpc GetIamPolicy(google.iam.v1.GetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + post: "/v3/{resource=tagValues/*}:getIamPolicy" + body: "*" + }; + option (google.api.method_signature) = "resource"; + } + + // Sets the access control policy on a TagValue, replacing any existing + // policy. The `resource` field should be the TagValue's resource name. + // For example: `tagValues/1234`. + // The caller must have `resourcemanager.tagValues.setIamPolicy` permission + // on the identified tagValue. + rpc SetIamPolicy(google.iam.v1.SetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + post: "/v3/{resource=tagValues/*}:setIamPolicy" + body: "*" + }; + option (google.api.method_signature) = "resource,policy"; + } + + // Returns permissions that a caller has on the specified TagValue. + // The `resource` field should be the TagValue's resource name. For example: + // `tagValues/1234`. + // + // There are no permissions required for making this API call. + rpc TestIamPermissions(google.iam.v1.TestIamPermissionsRequest) + returns (google.iam.v1.TestIamPermissionsResponse) { + option (google.api.http) = { + post: "/v3/{resource=tagValues/*}:testIamPermissions" + body: "*" + }; + option (google.api.method_signature) = "resource,permissions"; + } +} + +// A TagValue is a child of a particular TagKey. This is used to group +// cloud resources for the purpose of controlling them using policies. +message TagValue { + option (google.api.resource) = { + type: "cloudresourcemanager.googleapis.com/TagValue" + pattern: "tagValues/{tag_value}" + style: DECLARATIVE_FRIENDLY + }; + + // Immutable. Resource name for TagValue in the format `tagValues/456`. + string name = 1 [(google.api.field_behavior) = IMMUTABLE]; + + // Immutable. The resource name of the new TagValue's parent TagKey. + // Must be of the form `tagKeys/{tag_key_id}`. + string parent = 2 [(google.api.field_behavior) = IMMUTABLE]; + + // Required. Immutable. User-assigned short name for TagValue. The short name + // should be unique for TagValues within the same parent TagKey. + // + // The short name must be 63 characters or less, beginning and ending with + // an alphanumeric character ([a-z0-9A-Z]) with dashes (-), underscores (_), + // dots (.), and alphanumerics between. + string short_name = 3 [ + (google.api.field_behavior) = REQUIRED, + (google.api.field_behavior) = IMMUTABLE + ]; + + // Output only. The namespaced name of the TagValue. Can be in the form + // `{organization_id}/{tag_key_short_name}/{tag_value_short_name}` or + // `{project_id}/{tag_key_short_name}/{tag_value_short_name}` or + // `{project_number}/{tag_key_short_name}/{tag_value_short_name}`. + string namespaced_name = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. User-assigned description of the TagValue. + // Must not exceed 256 characters. + // + // Read-write. + string description = 5 [(google.api.field_behavior) = OPTIONAL]; + + // Output only. Creation time. + google.protobuf.Timestamp create_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Update time. + google.protobuf.Timestamp update_time = 7 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. Entity tag which users can pass to prevent race conditions. This + // field is always set in server responses. See UpdateTagValueRequest for + // details. + string etag = 8 [(google.api.field_behavior) = OPTIONAL]; +} + +// The request message for listing TagValues for the specified TagKey. +// Resource name for TagKey, parent of the TagValues to be listed, +// in the format `tagKeys/123`. +message ListTagValuesRequest { + // Required. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { child_type: "*" } + ]; + + // Optional. The maximum number of TagValues to return in the response. The + // server allows a maximum of 300 TagValues to return. If unspecified, the + // server will use 100 as the default. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to + // `ListTagValues` that indicates where this listing should continue from. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// The ListTagValues response. +message ListTagValuesResponse { + // A possibly paginated list of TagValues that are direct descendants of + // the specified parent TagKey. + repeated TagValue tag_values = 1; + + // A pagination token returned from a previous call to `ListTagValues` + // that indicates from where listing should continue. This is currently not + // used, but the server may at any point start supplying a valid token. + string next_page_token = 2; +} + +// The request message for getting a TagValue. +message GetTagValueRequest { + // Required. Resource name for TagValue to be fetched in the format + // `tagValues/456`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/TagValue" + } + ]; +} + +// The request message for getting a TagValue by its namespaced name. +message GetNamespacedTagValueRequest { + // Required. A namespaced tag value name in the following format: + // + // `{parentId}/{tagKeyShort}/{tagValueShort}` + // + // Examples: + // - `42/foo/abc` for a value with short name "abc" under the key with short + // name "foo" under the organization with ID 42 + // - `r2-d2/bar/xyz` for a value with short name "xyz" under the key with + // short name "bar" under the project with ID "r2-d2" + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/TagValue" + } + ]; +} + +// The request message for creating a TagValue. +message CreateTagValueRequest { + // Required. The TagValue to be created. Only fields `short_name`, + // `description`, and `parent` are considered during the creation request. + TagValue tag_value = 1 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Set as true to perform the validations necessary for creating the + // resource, but not actually perform the action. + bool validate_only = 2 [(google.api.field_behavior) = OPTIONAL]; +} + +// Runtime operation information for creating a TagValue. +message CreateTagValueMetadata {} + +// The request message for updating a TagValue. +message UpdateTagValueRequest { + // Required. The new definition of the TagValue. Only fields `description` and + // `etag` fields can be updated by this request. If the `etag` field is + // nonempty, it must match the `etag` field of the existing ControlGroup. + // Otherwise, `ABORTED` will be returned. + TagValue tag_value = 1 [(google.api.field_behavior) = REQUIRED]; + + // Optional. Fields to be updated. + google.protobuf.FieldMask update_mask = 2 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. True to perform validations necessary for updating the resource, + // but not actually perform the action. + bool validate_only = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// Runtime operation information for updating a TagValue. +message UpdateTagValueMetadata {} + +// The request message for deleting a TagValue. +message DeleteTagValueRequest { + // Required. Resource name for TagValue to be deleted in the format + // tagValues/456. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/TagValue" + } + ]; + + // Optional. Set as true to perform the validations necessary for deletion, + // but not actually perform the action. + bool validate_only = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The etag known to the client for the expected state of the + // TagValue. This is to be used for optimistic concurrency. + string etag = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// Runtime operation information for deleting a TagValue. +message DeleteTagValueMetadata {} diff --git a/proto/google/crypto/tink/BUILD b/proto/google/crypto/tink/BUILD new file mode 100644 index 0000000..8bf3ac3 --- /dev/null +++ b/proto/google/crypto/tink/BUILD @@ -0,0 +1,9 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "aes_gcm_proto", + srcs = ["aes_gcm.proto"], + strip_import_prefix = "/proto/", +) diff --git a/proto/google/crypto/tink/aes_gcm.proto b/proto/google/crypto/tink/aes_gcm.proto new file mode 100644 index 0000000..2551aa4 --- /dev/null +++ b/proto/google/crypto/tink/aes_gcm.proto @@ -0,0 +1,67 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +syntax = "proto3"; + +package google.crypto.tink; + +option java_package = "com.google.crypto.tink.proto"; +option java_multiple_files = true; +option go_package = "github.com/google/tink/go/proto/aes_gcm_go_proto"; +option objc_class_prefix = "TINKPB"; + +message AesGcmKeyFormat { + uint32 key_size = 2; + uint32 version = 3; +} + +// key_type: type.googleapis.com/google.crypto.tink.AesGcmKey +// +// A AesGcmKey is an AEAD key. Mathematically, it represents the functions +// Encrypt and Decrypt which we define in the following. +// +// First, Tink computes a "output prefix" OP by considering the +// "OutputPrefixType" message in Keyset.Key and the ID of the key using the +// Tink function "AEAD-OutputPrefix": (AesGcmKeys must always be stored in a +// keyset). +// +// AEAD-OutputPrefix(output_prefix_type, id): +// if output_prefix_type == RAW: +// return ""; +// if output_prefix_type == TINK: +// return 0x01 + BigEndian(id) +// if output_prefix_type == CRUNCHY: +// return 0x00 + BigEndian(id) +// +// Then, the function defined by this is defined as: +// [GCM], Section 5.2.1: +// * "Encrypt" maps a plaintext P and associated data A to a ciphertext given +// by the concatenation OP || IV || C || T. In addition to [GCM], Tink +// has the following restriction: IV is a uniformly random initialization +// vector of length 12 bytes and T is restricted to 16 bytes. +// +// * If OP matches the result of AEAD-OutputPrefix, then "Decrypt" maps the +// input OP || IV || C || T and A to the the output P in the manner as +// described in [GCM], Section 5.2.2. If OP does not match, then "Decrypt" +// returns an error. +// [GCM]: NIST Special Publication 800-38D: Recommendation for Block Cipher +// Modes of Operation: Galois/Counter Mode (GCM) and GMAC. +// http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf. + +message AesGcmKey { + uint32 version = 1; + bytes key_value = 3; +} diff --git a/proto/google/devtools/artifactregistry/v1/BUILD b/proto/google/devtools/artifactregistry/v1/BUILD new file mode 100644 index 0000000..d82b7b4 --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/BUILD @@ -0,0 +1,33 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "artifactregistry_proto", + srcs = [ + "apt_artifact.proto", + "artifact.proto", + "attachment.proto", + "file.proto", + "generic.proto", + "package.proto", + "repository.proto", + "rule.proto", + "service.proto", + "settings.proto", + "tag.proto", + "version.proto", + "vpcsc_config.proto", + "yum_artifact.proto", + ], + strip_import_prefix = "/proto/", + deps = [ + "//proto/google:com_google_protobuf", + "//proto/google:google_rpc", + "//proto/google:googleapis", + "//proto/google/iam/v1:iam_policy_proto", + "//proto/google/iam/v1:policy_proto", + "//proto/google/longrunning:operations_proto", + "//proto/google/type:expr_proto", + ], +) diff --git a/proto/google/devtools/artifactregistry/v1/apt_artifact.proto b/proto/google/devtools/artifactregistry/v1/apt_artifact.proto new file mode 100644 index 0000000..04c36d0 --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/apt_artifact.proto @@ -0,0 +1,114 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/rpc/status.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "AptArtifactProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// A detailed representation of an Apt artifact. Information in the record +// is derived from the archive's control file. +// See https://www.debian.org/doc/debian-policy/ch-controlfields.html +message AptArtifact { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/AptArtifact" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/aptArtifacts/{apt_artifact}" + }; + + // Package type is either binary or source. + enum PackageType { + // Package type is not specified. + PACKAGE_TYPE_UNSPECIFIED = 0; + + // Binary package. + BINARY = 1; + + // Source package. + SOURCE = 2; + } + + // Output only. The Artifact Registry resource name of the artifact. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The Apt package name of the artifact. + string package_name = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. An artifact is a binary or source package. + PackageType package_type = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Operating system architecture of the artifact. + string architecture = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Repository component of the artifact. + string component = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Contents of the artifact's control metadata file. + bytes control_file = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Google Cloud Storage location where the artifacts currently reside. +message ImportAptArtifactsGcsSource { + // Cloud Storage paths URI (e.g., gs://my_bucket//my_object). + repeated string uris = 1; + + // Supports URI wildcards for matching multiple objects from a single URI. + bool use_wildcards = 2; +} + +// The request to import new apt artifacts. +message ImportAptArtifactsRequest { + // The source location of the package binaries. + oneof source { + // Google Cloud Storage location where input content is located. + ImportAptArtifactsGcsSource gcs_source = 2; + } + + // The name of the parent resource where the artifacts will be imported. + string parent = 1; +} + +// Error information explaining why a package was not imported. +message ImportAptArtifactsErrorInfo { + // The source that was not imported. + oneof source { + // Google Cloud Storage location requested. + ImportAptArtifactsGcsSource gcs_source = 1; + } + + // The detailed error status. + google.rpc.Status error = 2; +} + +// The response message from importing APT artifacts. +message ImportAptArtifactsResponse { + // The Apt artifacts imported. + repeated AptArtifact apt_artifacts = 1; + + // Detailed error info for packages that were not imported. + repeated ImportAptArtifactsErrorInfo errors = 2; +} + +// The operation metadata for importing artifacts. +message ImportAptArtifactsMetadata {} diff --git a/proto/google/devtools/artifactregistry/v1/artifact.proto b/proto/google/devtools/artifactregistry/v1/artifact.proto new file mode 100644 index 0000000..c1895c7 --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/artifact.proto @@ -0,0 +1,355 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "ArtifactProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// DockerImage represents a docker artifact. +// The following fields are returned as untyped metadata in the Version +// resource, using camelcase keys (i.e. metadata.imageSizeBytes): +// * imageSizeBytes +// * mediaType +// * buildTime +message DockerImage { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/DockerImage" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/dockerImages/{docker_image}" + }; + + // Required. registry_location, project_id, repository_name and image id forms + // a unique image + // name:`projects//locations//repository//dockerImages/`. + // For example, + // "projects/test-project/locations/us-west4/repositories/test-repo/dockerImages/ + // nginx@sha256:e9954c1fc875017be1c3e36eca16be2d9e9bccc4bf072163515467d6a823c7cf", + // where "us-west4" is the registry_location, "test-project" is the + // project_id, "test-repo" is the repository_name and + // "nginx@sha256:e9954c1fc875017be1c3e36eca16be2d9e9bccc4bf072163515467d6a823c7cf" + // is the image's digest. + string name = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. URL to access the image. + // Example: + // us-west4-docker.pkg.dev/test-project/test-repo/nginx@sha256:e9954c1fc875017be1c3e36eca16be2d9e9bccc4bf072163515467d6a823c7cf + string uri = 2 [(google.api.field_behavior) = REQUIRED]; + + // Tags attached to this image. + repeated string tags = 3; + + // Calculated size of the image. + // This field is returned as the 'metadata.imageSizeBytes' field in the + // Version resource. + int64 image_size_bytes = 4; + + // Time the image was uploaded. + google.protobuf.Timestamp upload_time = 5; + + // Media type of this image, e.g. + // "application/vnd.docker.distribution.manifest.v2+json". + // This field is returned as the 'metadata.mediaType' field in the + // Version resource. + string media_type = 6; + + // The time this image was built. + // This field is returned as the 'metadata.buildTime' field in the + // Version resource. + // The build time is returned to the client as an RFC 3339 string, which can + // be easily used with the JavaScript Date constructor. + google.protobuf.Timestamp build_time = 7; + + // Output only. The time when the docker image was last updated. + google.protobuf.Timestamp update_time = 8 + [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// The request to list docker images. +message ListDockerImagesRequest { + // Required. The name of the parent resource whose docker images will be + // listed. + string parent = 1 [(google.api.field_behavior) = REQUIRED]; + + // The maximum number of artifacts to return. Maximum page size is 1,000. + int32 page_size = 2; + + // The next_page_token value returned from a previous list request, if any. + string page_token = 3; + + // The field to order the results by. + string order_by = 4; +} + +// The response from listing docker images. +message ListDockerImagesResponse { + // The docker images returned. + repeated DockerImage docker_images = 1; + + // The token to retrieve the next page of artifacts, or empty if there are no + // more artifacts to return. + string next_page_token = 2; +} + +// The request to get docker images. +message GetDockerImageRequest { + // Required. The name of the docker images. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/DockerImage" + } + ]; +} + +// MavenArtifact represents a maven artifact. +message MavenArtifact { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/MavenArtifact" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/mavenArtifacts/{maven_artifact}" + }; + + // Required. registry_location, project_id, repository_name and maven_artifact + // forms a unique artifact For example, + // "projects/test-project/locations/us-west4/repositories/test-repo/mavenArtifacts/ + // com.google.guava:guava:31.0-jre", + // where "us-west4" is the registry_location, "test-project" is the + // project_id, "test-repo" is the repository_name and + // "com.google.guava:guava:31.0-jre" + // is the maven artifact. + string name = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. URL to access the pom file of the artifact. + // Example: + // us-west4-maven.pkg.dev/test-project/test-repo/com/google/guava/guava/31.0/guava-31.0.pom + string pom_uri = 2 [(google.api.field_behavior) = REQUIRED]; + + // Group ID for the artifact. + // Example: + // com.google.guava + string group_id = 3; + + // Artifact ID for the artifact. + string artifact_id = 4; + + // Version of this artifact. + string version = 5; + + // Output only. Time the artifact was created. + google.protobuf.Timestamp create_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Time the artifact was updated. + google.protobuf.Timestamp update_time = 7 + [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// The request to list maven artifacts. +message ListMavenArtifactsRequest { + // Required. The name of the parent resource whose maven artifacts will be + // listed. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/MavenArtifact" + } + ]; + + // The maximum number of artifacts to return. Maximum page size is 1,000. + int32 page_size = 2; + + // The next_page_token value returned from a previous list request, if any. + string page_token = 3; +} + +// The response from listing maven artifacts. +message ListMavenArtifactsResponse { + // The maven artifacts returned. + repeated MavenArtifact maven_artifacts = 1; + + // The token to retrieve the next page of artifacts, or empty if there are no + // more artifacts to return. + string next_page_token = 2; +} + +// The request to get maven artifacts. +message GetMavenArtifactRequest { + // Required. The name of the maven artifact. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/MavenArtifact" + } + ]; +} + +// NpmPackage represents an npm artifact. +message NpmPackage { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/NpmPackage" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/npmPackages/{npm_package}" + }; + + // Required. registry_location, project_id, repository_name and npm_package + // forms a unique package For example, + // "projects/test-project/locations/us-west4/repositories/test-repo/npmPackages/ + // npm_test:1.0.0", + // where "us-west4" is the registry_location, "test-project" is the + // project_id, "test-repo" is the repository_name and + // npm_test:1.0.0" is the npm package. + string name = 1 [(google.api.field_behavior) = REQUIRED]; + + // Package for the artifact. + string package_name = 3; + + // Version of this package. + string version = 4; + + // Tags attached to this package. + repeated string tags = 5; + + // Output only. Time the package was created. + google.protobuf.Timestamp create_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Time the package was updated. + google.protobuf.Timestamp update_time = 7 + [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// The request to list npm packages. +message ListNpmPackagesRequest { + // Required. The name of the parent resource whose npm packages will be + // listed. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/NpmPackage" + } + ]; + + // The maximum number of artifacts to return. Maximum page size is 1,000. + int32 page_size = 2; + + // The next_page_token value returned from a previous list request, if any. + string page_token = 3; +} + +// The response from listing npm packages. +message ListNpmPackagesResponse { + // The npm packages returned. + repeated NpmPackage npm_packages = 1; + + // The token to retrieve the next page of artifacts, or empty if there are no + // more artifacts to return. + string next_page_token = 2; +} + +// The request to get npm packages. +message GetNpmPackageRequest { + // Required. The name of the npm package. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/NpmPackage" + } + ]; +} + +// PythonPackage represents a python artifact. +message PythonPackage { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/PythonPackage" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/pythonPackages/{python_package}" + }; + + // Required. registry_location, project_id, repository_name and python_package + // forms a unique package + // name:`projects//locations//repository//pythonPackages/`. + // For example, + // "projects/test-project/locations/us-west4/repositories/test-repo/pythonPackages/ + // python_package:1.0.0", + // where "us-west4" is the registry_location, "test-project" is the + // project_id, "test-repo" is the repository_name and + // python_package:1.0.0" is the python package. + string name = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. URL to access the package. + // Example: + // us-west4-python.pkg.dev/test-project/test-repo/python_package/file-name-1.0.0.tar.gz + string uri = 2 [(google.api.field_behavior) = REQUIRED]; + + // Package for the artifact. + string package_name = 3; + + // Version of this package. + string version = 4; + + // Output only. Time the package was created. + google.protobuf.Timestamp create_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Time the package was updated. + google.protobuf.Timestamp update_time = 7 + [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// The request to list python packages. +message ListPythonPackagesRequest { + // Required. The name of the parent resource whose python packages will be + // listed. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/PythonPackage" + } + ]; + + // The maximum number of artifacts to return. Maximum page size is 1,000. + int32 page_size = 2; + + // The next_page_token value returned from a previous list request, if any. + string page_token = 3; +} + +// The response from listing python packages. +message ListPythonPackagesResponse { + // The python packages returned. + repeated PythonPackage python_packages = 1; + + // The token to retrieve the next page of artifacts, or empty if there are no + // more artifacts to return. + string next_page_token = 2; +} + +// The request to get python packages. +message GetPythonPackageRequest { + // Required. The name of the python package. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/PythonPackage" + } + ]; +} diff --git a/proto/google/devtools/artifactregistry/v1/artifactregistry_grpc_service_config.json b/proto/google/devtools/artifactregistry/v1/artifactregistry_grpc_service_config.json new file mode 100644 index 0000000..2dfaa84 --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/artifactregistry_grpc_service_config.json @@ -0,0 +1,12 @@ +{ + "methodConfig": [ + { + "name": [ + { + "service": "google.devtools.artifactregistry.v1.ArtifactRegistry" + } + ], + "timeout": "60s" + } + ] +} diff --git a/proto/google/devtools/artifactregistry/v1/artifactregistry_v1.yaml b/proto/google/devtools/artifactregistry/v1/artifactregistry_v1.yaml new file mode 100644 index 0000000..23846cd --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/artifactregistry_v1.yaml @@ -0,0 +1,165 @@ +type: google.api.Service +config_version: 3 +name: artifactregistry.googleapis.com +title: Artifact Registry API + +apis: +- name: google.cloud.location.Locations +- name: google.devtools.artifactregistry.v1.ArtifactRegistry +- name: google.longrunning.Operations + +types: +- name: google.devtools.artifactregistry.v1.BatchDeleteVersionsMetadata +- name: google.devtools.artifactregistry.v1.GenericArtifact +- name: google.devtools.artifactregistry.v1.ImportAptArtifactsMetadata +- name: google.devtools.artifactregistry.v1.ImportAptArtifactsResponse +- name: google.devtools.artifactregistry.v1.ImportYumArtifactsMetadata +- name: google.devtools.artifactregistry.v1.ImportYumArtifactsResponse +- name: google.devtools.artifactregistry.v1.OperationMetadata + +documentation: + summary: |- + Store and manage build artifacts in a scalable and integrated service built + on Google infrastructure. + overview: |- + With Artifact Registry you can store and manage your build artifacts (e.g. + Docker images, Maven packages, npm packages), in a scalable and integrated + repository service built on Google infrastructure. You can manage + repository access + with IAM and interact with repositories via gcloud, Cloud Console, and + native package format tools. The service can also be integrated with Cloud + Build and other CI/CD systems. Artifact Registry abstracts away + infrastructure management, so you can focus on what matters most — + delivering value to the users of your services and applications. Note: + Enabling the Artifact Registry API will not affect your use of Container + Registry in the same project. + rules: + - selector: google.cloud.location.Locations.GetLocation + description: Gets information about a location. + + - selector: google.cloud.location.Locations.ListLocations + description: Lists information about the supported locations for this service. + +http: + rules: + - selector: google.cloud.location.Locations.GetLocation + get: '/v1/{name=projects/*/locations/*}' + - selector: google.cloud.location.Locations.ListLocations + get: '/v1/{name=projects/*}/locations' + - selector: google.longrunning.Operations.GetOperation + get: '/v1/{name=projects/*/locations/*/operations/*}' + +authentication: + rules: + - selector: google.cloud.location.Locations.GetLocation + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.cloud.location.Locations.ListLocations + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: 'google.devtools.artifactregistry.v1.ArtifactRegistry.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.BatchDeleteVersions + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.CreateAttachment + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.CreateRepository + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.CreateRule + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.CreateTag + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.DeleteAttachment + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.DeleteFile + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.DeletePackage + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.DeleteRepository + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.DeleteRule + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.DeleteTag + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.DeleteVersion + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.ImportAptArtifacts + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.ImportYumArtifacts + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.SetIamPolicy + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.UpdateFile + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.UpdatePackage + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.UpdateProjectSettings + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.UpdateRepository + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.UpdateRule + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.UpdateTag + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.UpdateVPCSCConfig + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.devtools.artifactregistry.v1.ArtifactRegistry.UpdateVersion + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform + - selector: google.longrunning.Operations.GetOperation + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/cloud-platform.read-only diff --git a/proto/google/devtools/artifactregistry/v1/artifactregistry_v1_grpc_service_config.json b/proto/google/devtools/artifactregistry/v1/artifactregistry_v1_grpc_service_config.json new file mode 100644 index 0000000..2dfaa84 --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/artifactregistry_v1_grpc_service_config.json @@ -0,0 +1,12 @@ +{ + "methodConfig": [ + { + "name": [ + { + "service": "google.devtools.artifactregistry.v1.ArtifactRegistry" + } + ], + "timeout": "60s" + } + ] +} diff --git a/proto/google/devtools/artifactregistry/v1/attachment.proto b/proto/google/devtools/artifactregistry/v1/attachment.proto new file mode 100644 index 0000000..a9c17ae --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/attachment.proto @@ -0,0 +1,160 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "AttachmentProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// An Attachment refers to additional metadata that can be attached to +// artifacts in Artifact Registry. An attachment consists of one or more files. +message Attachment { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/Attachment" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/attachments/{attachment}" + }; + + // The name of the attachment. E.g. + // `projects/p1/locations/us/repositories/repo/attachments/sbom`. + string name = 1; + + // Required. The target the attachment is for, can be a Version, Package or + // Repository. E.g. + // `projects/p1/locations/us-central1/repositories/repo1/packages/p1/versions/v1`. + string target = 2 [(google.api.field_behavior) = REQUIRED]; + + // Type of attachment. + // E.g. `application/vnd.spdx+json` + string type = 3; + + // The namespace this attachment belongs to. + // E.g. If an attachment is created by artifact analysis, namespace is set + // to `artifactanalysis.googleapis.com`. + string attachment_namespace = 4; + + // Optional. User annotations. These attributes can only be set and used by + // the user, and not by Artifact Registry. See + // https://google.aip.dev/128#annotations for more details such as format and + // size limitations. + map annotations = 5 [(google.api.field_behavior) = OPTIONAL]; + + // Output only. The time when the attachment was created. + google.protobuf.Timestamp create_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time when the attachment was last updated. + google.protobuf.Timestamp update_time = 7 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Required. The files that belong to this attachment. + // If the file ID part contains slashes, they are escaped. E.g. + // `projects/p1/locations/us-central1/repositories/repo1/files/sha:`. + repeated string files = 8 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/File" + } + ]; + + // Output only. The name of the OCI version that this attachment created. Only + // populated for Docker attachments. E.g. + // `projects/p1/locations/us-central1/repositories/repo1/packages/p1/versions/v1`. + string oci_version_name = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// The request to list attachments. +message ListAttachmentsRequest { + // Required. The name of the parent resource whose attachments will be listed. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/Attachment" + } + ]; + + // Optional. An expression for filtering the results of the request. Filter + // rules are case insensitive. The fields eligible for filtering are: + // + // * `target` + // * `type` + // * `attachment_namespace` + string filter = 2 [(google.api.field_behavior) = OPTIONAL]; + + // The maximum number of attachments to return. Maximum page size is 1,000. + int32 page_size = 3; + + // The next_page_token value returned from a previous list request, if any. + string page_token = 4; +} + +// The response from listing attachments. +message ListAttachmentsResponse { + // The attachments returned. + repeated Attachment attachments = 1; + + // The token to retrieve the next page of attachments, or empty if there are + // no more attachments to return. + string next_page_token = 2; +} + +// The request to retrieve an attachment. +message GetAttachmentRequest { + // Required. The name of the attachment to retrieve. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/Attachment" + } + ]; +} + +// The request to create a new attachment. +message CreateAttachmentRequest { + // Required. The name of the parent resource where the attachment will be + // created. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/Attachment" + } + ]; + + // Required. The attachment id to use for this attachment. + string attachment_id = 2 [(google.api.field_behavior) = REQUIRED]; + + // Required. The attachment to be created. + Attachment attachment = 3 [(google.api.field_behavior) = REQUIRED]; +} + +// The request to delete an attachment. +message DeleteAttachmentRequest { + // Required. The name of the attachment to delete. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/Attachment" + } + ]; +} diff --git a/proto/google/devtools/artifactregistry/v1/file.proto b/proto/google/devtools/artifactregistry/v1/file.proto new file mode 100644 index 0000000..0b19953 --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/file.proto @@ -0,0 +1,206 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "FileProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// A hash of file content. +message Hash { + // The algorithm used to compute the hash. + enum HashType { + // Unspecified. + HASH_TYPE_UNSPECIFIED = 0; + + // SHA256 hash. + SHA256 = 1; + + // MD5 hash. + MD5 = 2; + } + + // The algorithm used to compute the hash value. + HashType type = 1; + + // The hash value. + bytes value = 2; +} + +// Files store content that is potentially associated with Packages or Versions. +message File { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/File" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/files/{file}" + }; + + // The name of the file, for example: + // `projects/p1/locations/us-central1/repositories/repo1/files/a%2Fb%2Fc.txt`. + // If the file ID part contains slashes, they are escaped. + string name = 1; + + // The size of the File in bytes. + int64 size_bytes = 3; + + // The hashes of the file content. + repeated Hash hashes = 4; + + // Output only. The time when the File was created. + google.protobuf.Timestamp create_time = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time when the File was last updated. + google.protobuf.Timestamp update_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // The name of the Package or Version that owns this file, if any. + string owner = 7; + + // Output only. The time when the last attempt to refresh the file's data was + // made. Only set when the repository is remote. + google.protobuf.Timestamp fetch_time = 8 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. Client specified annotations. + map annotations = 9 [(google.api.field_behavior) = OPTIONAL]; +} + +// The request to list files. +message ListFilesRequest { + // Required. The name of the repository whose files will be listed. For + // example: "projects/p1/locations/us-central1/repositories/repo1 + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/File" + } + ]; + + // An expression for filtering the results of the request. Filter rules are + // case insensitive. The fields eligible for filtering are: + // + // * `name` + // * `owner` + // * `annotations` + // + // Examples of using a filter: + // + // To filter the results of your request to files with the name `my_file.txt` + // in project `my-project` in the `us-central` region, in repository + // `my-repo`, append the following filter expression to your request: + // + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/files/my-file.txt"` + // + // You can also use wildcards to match any number of characters before or + // after the value: + // + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/files/my-*"` + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/files/*file.txt"` + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/files/*file*"` + // + // To filter the results of your request to files owned by the version `1.0` + // in package `pkg1`, append the following filter expression to your request: + // + // * `owner="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-package/versions/1.0"` + // + // To filter the results of your request to files with the annotation + // key-value pair [`external_link`: `external_link_value`], append the + // following filter expression to your request: + // + // * `"annotations.external_link:external_link_value"` + // + // To filter just for a specific annotation key `external_link`, append the + // following filter expression to your request: + // + // * `"annotations.external_link"` + // + // If the annotation key or value contains special characters, you can escape + // them by surrounding the value with backticks. For example, to filter the + // results of your request to files with the annotation key-value pair + // [`external.link`:`https://example.com/my-file`], append the following + // filter expression to your request: + // + // * `` "annotations.`external.link`:`https://example.com/my-file`" `` + // + // You can also filter with annotations with a wildcard to + // match any number of characters before or after the value: + // + // * `` "annotations.*_link:`*example.com*`" `` + string filter = 4; + + // The maximum number of files to return. Maximum page size is 1,000. + int32 page_size = 2; + + // The next_page_token value returned from a previous list request, if any. + string page_token = 3; + + // The field to order the results by. + string order_by = 5; +} + +// The response from listing files. +message ListFilesResponse { + // The files returned. + repeated File files = 1; + + // The token to retrieve the next page of files, or empty if there are no + // more files to return. + string next_page_token = 2; +} + +// The request to retrieve a file. +message GetFileRequest { + // Required. The name of the file to retrieve. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/File" + } + ]; +} + +// The request to delete a file. +message DeleteFileRequest { + // Required. The name of the file to delete. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/File" + } + ]; +} + +// The request to update a file. +message UpdateFileRequest { + // Required. The File that replaces the resource on the server. + File file = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. The update mask applies to the resource. For the `FieldMask` + // definition, see + // https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#fieldmask + google.protobuf.FieldMask update_mask = 2 + [(google.api.field_behavior) = REQUIRED]; +} diff --git a/proto/google/devtools/artifactregistry/v1/generic.proto b/proto/google/devtools/artifactregistry/v1/generic.proto new file mode 100644 index 0000000..64b71bc --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/generic.proto @@ -0,0 +1,55 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "GenericProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// GenericArtifact represents a generic artifact +message GenericArtifact { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/GenericArtifact" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/genericArtifacts/{generic_artifact}" + }; + + // Resource name of the generic artifact. + // project, location, repository, package_id and version_id + // create a unique generic artifact. + // i.e. "projects/test-project/locations/us-west4/repositories/test-repo/ + // genericArtifacts/package_id:version_id" + string name = 1; + + // The version of the generic artifact. + string version = 2; + + // Output only. The time when the Generic module is created. + google.protobuf.Timestamp create_time = 3 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time when the Generic module is updated. + google.protobuf.Timestamp update_time = 4 + [(google.api.field_behavior) = OUTPUT_ONLY]; +} diff --git a/proto/google/devtools/artifactregistry/v1/package.proto b/proto/google/devtools/artifactregistry/v1/package.proto new file mode 100644 index 0000000..27cddcf --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/package.proto @@ -0,0 +1,166 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "PackageProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// Packages are named collections of versions. +message Package { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/Package" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/packages/{package}" + }; + + // The name of the package, for example: + // `projects/p1/locations/us-central1/repositories/repo1/packages/pkg1`. + // If the package ID part contains slashes, the slashes are escaped. + string name = 1; + + // The display name of the package. + string display_name = 2; + + // The time when the package was created. + google.protobuf.Timestamp create_time = 5; + + // The time when the package was last updated. This includes publishing a new + // version of the package. + google.protobuf.Timestamp update_time = 6; + + // Optional. Client specified annotations. + map annotations = 7 [(google.api.field_behavior) = OPTIONAL]; +} + +// The request to list packages. +message ListPackagesRequest { + // Required. The name of the parent resource whose packages will be listed. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/Package" + } + ]; + + // The maximum number of packages to return. Maximum page size is 1,000. + int32 page_size = 2; + + // The next_page_token value returned from a previous list request, if any. + string page_token = 3; + + // Optional. An expression for filtering the results of the request. Filter + // rules are case insensitive. The fields eligible for filtering are: + // + // * `name` + // * `annotations` + // + // Examples of using a filter: + // + // To filter the results of your request to packages with the name + // `my-package` in project `my-project` in the `us-central` region, in + // repository `my-repo`, append the following filter expression to your + // request: + // + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-package"` + // + // You can also use wildcards to match any number of characters before or + // after the value: + // + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-*"` + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/*package"` + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/*pack*"` + // + // To filter the results of your request to packages with the annotation + // key-value pair [`external_link`: `external_link_value`], append the + // following filter expression to your request": + // + // * `"annotations.external_link:external_link_value"` + // + // To filter the results just for a specific annotation key `external_link`, + // append the following filter expression to your request: + // + // * `"annotations.external_link"` + // + // If the annotation key or value contains special characters, you can escape + // them by surrounding the value with backticks. For example, to filter the + // results of your request to packages with the annotation key-value pair + // [`external.link`:`https://example.com/my-package`], append the following + // filter expression to your request: + // + // * `` "annotations.`external.link`:`https://example.com/my-package`" `` + // + // You can also filter with annotations with a wildcard to + // match any number of characters before or after the value: + // + // * `` "annotations.*_link:`*example.com*`" `` + string filter = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The field to order the results by. + string order_by = 5 [(google.api.field_behavior) = OPTIONAL]; +} + +// The response from listing packages. +message ListPackagesResponse { + // The packages returned. + repeated Package packages = 1; + + // The token to retrieve the next page of packages, or empty if there are no + // more packages to return. + string next_page_token = 2; +} + +// The request to retrieve a package. +message GetPackageRequest { + // Required. The name of the package to retrieve. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/Package" + } + ]; +} + +// The request to delete a package. +message DeletePackageRequest { + // Required. The name of the package to delete. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/Package" + } + ]; +} + +// The request to update a package. +message UpdatePackageRequest { + // The package that replaces the resource on the server. + Package package = 1; + + // The update mask applies to the resource. For the `FieldMask` definition, + // see + // https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#fieldmask + google.protobuf.FieldMask update_mask = 2; +} diff --git a/proto/google/devtools/artifactregistry/v1/repository.proto b/proto/google/devtools/artifactregistry/v1/repository.proto new file mode 100644 index 0000000..9ef8180 --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/repository.proto @@ -0,0 +1,757 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "RepositoryProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; +option (google.api.resource_definition) = { + type: "secretmanager.googleapis.com/SecretVersion" + pattern: "projects/{project}/secrets/{secret}/versions/{secret_version}" +}; +option (google.api.resource_definition) = { + type: "servicedirectory.googleapis.com/Service" + pattern: "projects/{project}/locations/{location}/namespaces/{namespace}/services/{service}" +}; + +// Artifact policy configuration for the repository contents. +message UpstreamPolicy { + // The user-provided ID of the upstream policy. + string id = 1; + + // A reference to the repository resource, for example: + // `projects/p1/locations/us-central1/repositories/repo1`. + string repository = 2 [(google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/Repository" + }]; + + // Entries with a greater priority value take precedence in the pull order. + int32 priority = 3; +} + +// CleanupPolicyCondition is a set of conditions attached to a CleanupPolicy. +// If multiple entries are set, all must be satisfied for the condition to be +// satisfied. +message CleanupPolicyCondition { + // Statuses applying to versions. + enum TagState { + // Tag status not specified. + TAG_STATE_UNSPECIFIED = 0; + + // Applies to tagged versions only. + TAGGED = 1; + + // Applies to untagged versions only. + UNTAGGED = 2; + + // Applies to all versions. + ANY = 3; + } + + // Match versions by tag status. + optional TagState tag_state = 2; + + // Match versions by tag prefix. Applied on any prefix match. + repeated string tag_prefixes = 3; + + // Match versions by version name prefix. Applied on any prefix match. + repeated string version_name_prefixes = 4; + + // Match versions by package prefix. Applied on any prefix match. + repeated string package_name_prefixes = 5; + + // Match versions older than a duration. + optional google.protobuf.Duration older_than = 6; + + // Match versions newer than a duration. + optional google.protobuf.Duration newer_than = 7; +} + +// CleanupPolicyMostRecentVersions is an alternate condition of a CleanupPolicy +// for retaining a minimum number of versions. +message CleanupPolicyMostRecentVersions { + // List of package name prefixes that will apply this rule. + repeated string package_name_prefixes = 1; + + // Minimum number of versions to keep. + optional int32 keep_count = 2; +} + +// Artifact policy configuration for repository cleanup policies. +message CleanupPolicy { + // Action type for a cleanup policy. + enum Action { + // Action not specified. + ACTION_UNSPECIFIED = 0; + + // Delete action. + DELETE = 1; + + // Keep action. + KEEP = 2; + } + + oneof condition_type { + // Policy condition for matching versions. + CleanupPolicyCondition condition = 2; + + // Policy condition for retaining a minimum number of versions. May only be + // specified with a Keep action. + CleanupPolicyMostRecentVersions most_recent_versions = 4; + } + + // The user-provided ID of the cleanup policy. + string id = 1; + + // Policy action. + Action action = 3; +} + +// Virtual repository configuration. +message VirtualRepositoryConfig { + // Policies that configure the upstream artifacts distributed by the Virtual + // Repository. Upstream policies cannot be set on a standard repository. + repeated UpstreamPolicy upstream_policies = 1; +} + +// Remote repository configuration. +message RemoteRepositoryConfig { + // The credentials to access the remote repository. + message UpstreamCredentials { + // Username and password credentials. + message UsernamePasswordCredentials { + // The username to access the remote repository. + string username = 1; + + // The Secret Manager key version that holds the password to access the + // remote repository. Must be in the format of + // `projects/{project}/secrets/{secret}/versions/{version}`. + string password_secret_version = 2 [(google.api.resource_reference) = { + type: "secretmanager.googleapis.com/SecretVersion" + }]; + } + + oneof credentials { + // Use username and password to access the remote repository. + UsernamePasswordCredentials username_password_credentials = 1; + } + } + + // Configuration for a Docker remote repository. + message DockerRepository { + // Customer-specified publicly available remote repository. + message CustomRepository { + // An http/https uri reference to the custom remote repository, for ex: + // "https://registry-1.docker.io". + string uri = 1; + } + + // Predefined list of publicly available Docker repositories like Docker + // Hub. + enum PublicRepository { + // Unspecified repository. + PUBLIC_REPOSITORY_UNSPECIFIED = 0; + + // Docker Hub. + DOCKER_HUB = 1; + } + + // Address of the remote repository. + oneof upstream { + // One of the publicly available Docker repositories supported by Artifact + // Registry. + PublicRepository public_repository = 1; + + // Customer-specified remote repository. + CustomRepository custom_repository = 3; + } + } + + // Configuration for a Maven remote repository. + message MavenRepository { + // Customer-specified publicly available remote repository. + message CustomRepository { + // An http/https uri reference to the upstream remote repository, for ex: + // "https://my.maven.registry/". + string uri = 1; + } + + // Predefined list of publicly available Maven repositories like Maven + // Central. + enum PublicRepository { + // Unspecified repository. + PUBLIC_REPOSITORY_UNSPECIFIED = 0; + + // Maven Central. + MAVEN_CENTRAL = 1; + } + + // Address of the remote repository. + oneof upstream { + // One of the publicly available Maven repositories supported by Artifact + // Registry. + PublicRepository public_repository = 1; + + // Customer-specified remote repository. + CustomRepository custom_repository = 3; + } + } + + // Configuration for a Npm remote repository. + message NpmRepository { + // Customer-specified publicly available remote repository. + message CustomRepository { + // An http/https uri reference to the upstream remote repository, for ex: + // "https://my.npm.registry/". + string uri = 1; + } + + // Predefined list of publicly available NPM repositories like npmjs. + enum PublicRepository { + // Unspecified repository. + PUBLIC_REPOSITORY_UNSPECIFIED = 0; + + // npmjs. + NPMJS = 1; + } + + // Address of the remote repository + oneof upstream { + // One of the publicly available Npm repositories supported by Artifact + // Registry. + PublicRepository public_repository = 1; + + // Customer-specified remote repository. + CustomRepository custom_repository = 3; + } + } + + // Configuration for a Python remote repository. + message PythonRepository { + // Customer-specified publicly available remote repository. + message CustomRepository { + // An http/https uri reference to the upstream remote repository, for ex: + // "https://my.python.registry/". + string uri = 1; + } + + // Predefined list of publicly available Python repositories like PyPI.org. + enum PublicRepository { + // Unspecified repository. + PUBLIC_REPOSITORY_UNSPECIFIED = 0; + + // PyPI. + PYPI = 1; + } + + // Address of the remote repository. + oneof upstream { + // One of the publicly available Python repositories supported by Artifact + // Registry. + PublicRepository public_repository = 1; + + // Customer-specified remote repository. + CustomRepository custom_repository = 3; + } + } + + // Configuration for an Apt remote repository. + message AptRepository { + // Publicly available Apt repositories constructed from a common repository + // base and a custom repository path. + message PublicRepository { + // Predefined list of publicly available repository bases for Apt. + enum RepositoryBase { + // Unspecified repository base. + REPOSITORY_BASE_UNSPECIFIED = 0; + + // Debian. + DEBIAN = 1; + + // Ubuntu LTS/Pro. + UBUNTU = 2; + + // Archived Debian. + DEBIAN_SNAPSHOT = 3; + } + + // A common public repository base for Apt. + RepositoryBase repository_base = 1; + + // A custom field to define a path to a specific repository from the base. + string repository_path = 2; + } + + // Customer-specified publicly available remote repository. + message CustomRepository { + // An http/https uri reference to the upstream remote repository, for ex: + // "https://my.apt.registry/". + string uri = 1; + } + + // Address of the remote repository. + oneof upstream { + // One of the publicly available Apt repositories supported by Artifact + // Registry. + PublicRepository public_repository = 1; + + // Customer-specified remote repository. + CustomRepository custom_repository = 3; + } + } + + // Configuration for a Yum remote repository. + message YumRepository { + // Publicly available Yum repositories constructed from a common repository + // base and a custom repository path. + message PublicRepository { + // Predefined list of publicly available repository bases for Yum. + enum RepositoryBase { + // Unspecified repository base. + REPOSITORY_BASE_UNSPECIFIED = 0; + + // CentOS. + CENTOS = 1; + + // CentOS Debug. + CENTOS_DEBUG = 2; + + // CentOS Vault. + CENTOS_VAULT = 3; + + // CentOS Stream. + CENTOS_STREAM = 4; + + // Rocky. + ROCKY = 5; + + // Fedora Extra Packages for Enterprise Linux (EPEL). + EPEL = 6; + } + + // A common public repository base for Yum. + RepositoryBase repository_base = 1; + + // A custom field to define a path to a specific repository from the base. + string repository_path = 2; + } + + // Customer-specified publicly available remote repository. + message CustomRepository { + // An http/https uri reference to the upstream remote repository, for ex: + // "https://my.yum.registry/". + string uri = 1; + } + + // Address of the remote repository. + oneof upstream { + // One of the publicly available Yum repositories supported by Artifact + // Registry. + PublicRepository public_repository = 1; + + // Customer-specified remote repository. + CustomRepository custom_repository = 3; + } + } + + // Common remote repository settings type. + message CommonRemoteRepository { + // Required. A common public repository base for remote repository. + string uri = 1 [(google.api.field_behavior) = REQUIRED]; + } + + // Settings specific to the remote repository. + oneof remote_source { + // Specific settings for a Docker remote repository. + DockerRepository docker_repository = 2; + + // Specific settings for a Maven remote repository. + MavenRepository maven_repository = 3; + + // Specific settings for an Npm remote repository. + NpmRepository npm_repository = 4; + + // Specific settings for a Python remote repository. + PythonRepository python_repository = 5; + + // Specific settings for an Apt remote repository. + AptRepository apt_repository = 6; + + // Specific settings for a Yum remote repository. + YumRepository yum_repository = 7; + + // Common remote repository settings. + // Used as the remote repository upstream URL. + CommonRemoteRepository common_repository = 14; + } + + // The description of the remote source. + string description = 1; + + // Optional. The credentials used to access the remote repository. + UpstreamCredentials upstream_credentials = 9 + [(google.api.field_behavior) = OPTIONAL]; + + // Input only. A create/update remote repo option to avoid making a HEAD/GET + // request to validate a remote repo and any supplied upstream credentials. + bool disable_upstream_validation = 12 + [(google.api.field_behavior) = INPUT_ONLY]; +} + +// A Repository for storing artifacts with a specific format. +message Repository { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/Repository" + pattern: "projects/{project}/locations/{location}/repositories/{repository}" + }; + + // MavenRepositoryConfig is maven related repository details. + // Provides additional configuration details for repositories of the maven + // format type. + message MavenRepositoryConfig { + // VersionPolicy is the version policy for the repository. + enum VersionPolicy { + // VERSION_POLICY_UNSPECIFIED - the version policy is not defined. + // When the version policy is not defined, no validation is performed + // for the versions. + VERSION_POLICY_UNSPECIFIED = 0; + + // RELEASE - repository will accept only Release versions. + RELEASE = 1; + + // SNAPSHOT - repository will accept only Snapshot versions. + SNAPSHOT = 2; + } + + // The repository with this flag will allow publishing + // the same snapshot versions. + bool allow_snapshot_overwrites = 1; + + // Version policy defines the versions that the registry will accept. + VersionPolicy version_policy = 2; + } + + // DockerRepositoryConfig is docker related repository details. + // Provides additional configuration details for repositories of the docker + // format type. + message DockerRepositoryConfig { + // The repository which enabled this flag prevents all tags from being + // modified, moved or deleted. This does not prevent tags from being + // created. + bool immutable_tags = 1; + } + + // Config on whether to perform vulnerability scanning for resources in this + // repository, as well as output fields describing current state. + message VulnerabilityScanningConfig { + // Config for vulnerability scanning of resources in this repository. + enum EnablementConfig { + // Not set. This will be treated as INHERITED. + ENABLEMENT_CONFIG_UNSPECIFIED = 0; + + // Scanning is Enabled, but dependent on API enablement. + INHERITED = 1; + + // No automatic vulnerability scanning will be performed for this + // repository. + DISABLED = 2; + } + + // Describes the state of vulnerability scanning in this repository, + // including both repository enablement and API enablement. + enum EnablementState { + // Enablement state is unclear. + ENABLEMENT_STATE_UNSPECIFIED = 0; + + // Repository does not support vulnerability scanning. + SCANNING_UNSUPPORTED = 1; + + // Vulnerability scanning is disabled for this repository. + SCANNING_DISABLED = 2; + + // Vulnerability scanning is active for this repository. + SCANNING_ACTIVE = 3; + } + + // Optional. Config for whether this repository has vulnerability scanning + // disabled. + EnablementConfig enablement_config = 1 + [(google.api.field_behavior) = OPTIONAL]; + + // Output only. The last time this repository config was enabled. + google.protobuf.Timestamp last_enable_time = 2 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. State of feature enablement, combining repository enablement + // config and API enablement state. + EnablementState enablement_state = 3 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Reason for the repository state. + string enablement_state_reason = 4 + [(google.api.field_behavior) = OUTPUT_ONLY]; + } + + // A package format. + enum Format { + // Unspecified package format. + FORMAT_UNSPECIFIED = 0; + + // Docker package format. + DOCKER = 1; + + // Maven package format. + MAVEN = 2; + + // NPM package format. + NPM = 3; + + // APT package format. + APT = 5; + + // YUM package format. + YUM = 6; + + // Python package format. + PYTHON = 8; + + // Kubeflow Pipelines package format. + KFP = 9; + + // Go package format. + GO = 10; + + // Generic package format. + GENERIC = 11; + } + + // The mode configures the repository to serve artifacts from different + // sources. + enum Mode { + // Unspecified mode. + MODE_UNSPECIFIED = 0; + + // A standard repository storing artifacts. + STANDARD_REPOSITORY = 1; + + // A virtual repository to serve artifacts from one or more sources. + VIRTUAL_REPOSITORY = 2; + + // A remote repository to serve artifacts from a remote source. + REMOTE_REPOSITORY = 3; + } + + // Repository-specific configurations. + oneof format_config { + // Maven repository config contains repository level configuration + // for the repositories of maven type. + MavenRepositoryConfig maven_config = 9; + + // Docker repository config contains repository level configuration + // for the repositories of docker type. + DockerRepositoryConfig docker_config = 17; + } + + // Repository configuration specific to the Mode value being selected (Remote + // or Virtual) + oneof mode_config { + // Configuration specific for a Virtual Repository. + VirtualRepositoryConfig virtual_repository_config = 14; + + // Configuration specific for a Remote Repository. + RemoteRepositoryConfig remote_repository_config = 15; + } + + // The name of the repository, for example: + // `projects/p1/locations/us-central1/repositories/repo1`. For each location + // in a project, repository names must be unique. + string name = 1; + + // Optional. The format of packages that are stored in the repository. + Format format = 2 [(google.api.field_behavior) = OPTIONAL]; + + // The user-provided description of the repository. + string description = 3; + + // Labels with user-defined metadata. + // This field may contain up to 64 entries. Label keys and values may be no + // longer than 63 characters. Label keys must begin with a lowercase letter + // and may only contain lowercase letters, numeric characters, underscores, + // and dashes. + map labels = 4; + + // Output only. The time when the repository was created. + google.protobuf.Timestamp create_time = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The time when the repository was last updated. + google.protobuf.Timestamp update_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // The Cloud KMS resource name of the customer managed encryption key that's + // used to encrypt the contents of the Repository. Has the form: + // `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. + // This value may not be changed after the Repository has been created. + string kms_key_name = 8; + + // Optional. The mode of the repository. + Mode mode = 10 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Cleanup policies for this repository. Cleanup policies indicate + // when certain package versions can be automatically deleted. Map keys are + // policy IDs supplied by users during policy creation. They must unique + // within a repository and be under 128 characters in length. + map cleanup_policies = 12 + [(google.api.field_behavior) = OPTIONAL]; + + // Output only. The size, in bytes, of all artifact storage in this + // repository. Repositories that are generally available or in public preview + // use this to calculate storage costs. + int64 size_bytes = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. If set, the repository satisfies physical zone separation. + bool satisfies_pzs = 16 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. If true, the cleanup pipeline is prevented from deleting versions + // in this repository. + bool cleanup_policy_dry_run = 18 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Config and state for vulnerability scanning of resources within + // this Repository. + VulnerabilityScanningConfig vulnerability_scanning_config = 19 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. If this is true, an unspecified repo type will be treated as + // error rather than defaulting to standard. + bool disallow_unspecified_mode = 21 [(google.api.field_behavior) = OPTIONAL]; + + // Output only. If set, the repository satisfies physical zone isolation. + bool satisfies_pzi = 22 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// The request to list repositories. +message ListRepositoriesRequest { + // Required. The name of the parent resource whose repositories will be + // listed. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/Repository" + } + ]; + + // The maximum number of repositories to return. Maximum page size is 1,000. + int32 page_size = 2; + + // The next_page_token value returned from a previous list request, if any. + string page_token = 3; + + // Optional. An expression for filtering the results of the request. Filter + // rules are case insensitive. The fields eligible for filtering are: + // + // * `name` + // + // Examples of using a filter: + // + // To filter the results of your request to repositories with the name + // `my-repo` in project `my-project` in the `us-central` region, append the + // following filter expression to your request: + // + // * `name="projects/my-project/locations/us-central1/repositories/my-repo"` + // + // You can also use wildcards to match any number of characters before or + // after the value: + // + // * `name="projects/my-project/locations/us-central1/repositories/my-*"` + // * `name="projects/my-project/locations/us-central1/repositories/*repo"` + // * `name="projects/my-project/locations/us-central1/repositories/*repo*"` + string filter = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The field to order the results by. + string order_by = 5 [(google.api.field_behavior) = OPTIONAL]; +} + +// The response from listing repositories. +message ListRepositoriesResponse { + // The repositories returned. + repeated Repository repositories = 1; + + // The token to retrieve the next page of repositories, or empty if there are + // no more repositories to return. + string next_page_token = 2; +} + +// The request to retrieve a repository. +message GetRepositoryRequest { + // Required. The name of the repository to retrieve. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/Repository" + } + ]; +} + +// The request to create a new repository. +message CreateRepositoryRequest { + // Required. The name of the parent resource where the repository will be + // created. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/Repository" + } + ]; + + // Required. The repository id to use for this repository. + string repository_id = 2 [(google.api.field_behavior) = REQUIRED]; + + // Required. The repository to be created. + Repository repository = 3 [(google.api.field_behavior) = REQUIRED]; +} + +// The request to update a repository. +message UpdateRepositoryRequest { + // The repository that replaces the resource on the server. + Repository repository = 1; + + // The update mask applies to the resource. For the `FieldMask` definition, + // see + // https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#fieldmask + google.protobuf.FieldMask update_mask = 2; +} + +// The request to delete a repository. +message DeleteRepositoryRequest { + // Required. The name of the repository to delete. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/Repository" + } + ]; +} diff --git a/proto/google/devtools/artifactregistry/v1/rule.proto b/proto/google/devtools/artifactregistry/v1/rule.proto new file mode 100644 index 0000000..9d1d5b2 --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/rule.proto @@ -0,0 +1,157 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/field_mask.proto"; +import "google/type/expr.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "RuleProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// A rule defines the deny or allow action of the operation it applies to and +// the conditions required for the rule to apply. You can set one rule for an +// entire repository and one rule for each package within. +message Rule { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/Rule" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/rules/{rule}" + }; + + // Defines the action of the rule. + enum Action { + // Action not specified. + ACTION_UNSPECIFIED = 0; + + // Allow the operation. + ALLOW = 1; + + // Deny the operation. + DENY = 2; + } + + // The operation the rule applies to. + enum Operation { + // Operation not specified. + OPERATION_UNSPECIFIED = 0; + + // Download operation. + DOWNLOAD = 1; + } + + // The name of the rule, for example: + // `projects/p1/locations/us-central1/repositories/repo1/rules/rule1`. + string name = 1; + + // The action this rule takes. + Action action = 2; + + Operation operation = 3; + + // Optional. A CEL expression for conditions that must be met in order for the + // rule to apply. If not provided, the rule matches all objects. + google.type.Expr condition = 4 [(google.api.field_behavior) = OPTIONAL]; + + // The package ID the rule applies to. + // If empty, this rule applies to all packages inside the repository. + string package_id = 5; +} + +// The request to list rules. +message ListRulesRequest { + // Required. The name of the parent repository whose rules will be listed. + // For example: + // `projects/p1/locations/us-central1/repositories/repo1`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/Rule" + } + ]; + + // The maximum number of rules to return. Maximum page size is 1,000. + int32 page_size = 2; + + // The next_page_token value returned from a previous list request, if any. + string page_token = 3; +} + +// The response from listing rules. +message ListRulesResponse { + // The rules returned. + repeated Rule rules = 1; + + // The token to retrieve the next page of rules, or empty if there are no + // more rules to return. + string next_page_token = 2; +} + +// The request to retrieve a rule. +message GetRuleRequest { + // Required. The name of the rule to retrieve. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/Rule" + } + ]; +} + +// The request to create a new rule. +message CreateRuleRequest { + // Required. The name of the parent resource where the rule will be created. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/Rule" + } + ]; + + // The rule id to use for this repository. + string rule_id = 2; + + // The rule to be created. + Rule rule = 3; +} + +// The request to update a rule. +message UpdateRuleRequest { + // The rule that replaces the resource on the server. + Rule rule = 1; + + // The update mask applies to the resource. For the `FieldMask` definition, + // see + // https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#fieldmask + google.protobuf.FieldMask update_mask = 2; +} + +// The request to delete a rule. +message DeleteRuleRequest { + // Required. The name of the rule to delete. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/Rule" + } + ]; +} diff --git a/proto/google/devtools/artifactregistry/v1/service.proto b/proto/google/devtools/artifactregistry/v1/service.proto new file mode 100644 index 0000000..29daf6c --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/service.proto @@ -0,0 +1,547 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/devtools/artifactregistry/v1/apt_artifact.proto"; +import "google/devtools/artifactregistry/v1/artifact.proto"; +import "google/devtools/artifactregistry/v1/attachment.proto"; +import "google/devtools/artifactregistry/v1/file.proto"; +import "google/devtools/artifactregistry/v1/package.proto"; +import "google/devtools/artifactregistry/v1/repository.proto"; +import "google/devtools/artifactregistry/v1/rule.proto"; +import "google/devtools/artifactregistry/v1/settings.proto"; +import "google/devtools/artifactregistry/v1/tag.proto"; +import "google/devtools/artifactregistry/v1/version.proto"; +import "google/devtools/artifactregistry/v1/vpcsc_config.proto"; +import "google/devtools/artifactregistry/v1/yum_artifact.proto"; +import "google/iam/v1/iam_policy.proto"; +import "google/iam/v1/policy.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/empty.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "ServiceProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// The Artifact Registry API service. +// +// Artifact Registry is an artifact management system for storing artifacts +// from different package management systems. +// +// The resources managed by this API are: +// +// * Repositories, which group packages and their data. +// * Packages, which group versions and their tags. +// * Versions, which are specific forms of a package. +// * Tags, which represent alternative names for versions. +// * Files, which contain content and are optionally associated with a Package +// or Version. +service ArtifactRegistry { + option (google.api.default_host) = "artifactregistry.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/cloud-platform.read-only"; + + // Lists docker images. + rpc ListDockerImages(ListDockerImagesRequest) + returns (ListDockerImagesResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/repositories/*}/dockerImages" + }; + option (google.api.method_signature) = "parent"; + } + + // Gets a docker image. + rpc GetDockerImage(GetDockerImageRequest) returns (DockerImage) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/repositories/*/dockerImages/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Lists maven artifacts. + rpc ListMavenArtifacts(ListMavenArtifactsRequest) + returns (ListMavenArtifactsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/repositories/*}/mavenArtifacts" + }; + option (google.api.method_signature) = "parent"; + } + + // Gets a maven artifact. + rpc GetMavenArtifact(GetMavenArtifactRequest) returns (MavenArtifact) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/repositories/*/mavenArtifacts/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Lists npm packages. + rpc ListNpmPackages(ListNpmPackagesRequest) + returns (ListNpmPackagesResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/repositories/*}/npmPackages" + }; + option (google.api.method_signature) = "parent"; + } + + // Gets a npm package. + rpc GetNpmPackage(GetNpmPackageRequest) returns (NpmPackage) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/repositories/*/npmPackages/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Lists python packages. + rpc ListPythonPackages(ListPythonPackagesRequest) + returns (ListPythonPackagesResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/repositories/*}/pythonPackages" + }; + option (google.api.method_signature) = "parent"; + } + + // Gets a python package. + rpc GetPythonPackage(GetPythonPackageRequest) returns (PythonPackage) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/repositories/*/pythonPackages/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Imports Apt artifacts. The returned Operation will complete once the + // resources are imported. Package, Version, and File resources are created + // based on the imported artifacts. Imported artifacts that conflict with + // existing resources are ignored. + rpc ImportAptArtifacts(ImportAptArtifactsRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/repositories/*}/aptArtifacts:import" + body: "*" + }; + option (google.longrunning.operation_info) = { + response_type: "google.devtools.artifactregistry.v1.ImportAptArtifactsResponse" + metadata_type: "google.devtools.artifactregistry.v1.ImportAptArtifactsMetadata" + }; + } + + // Imports Yum (RPM) artifacts. The returned Operation will complete once the + // resources are imported. Package, Version, and File resources are created + // based on the imported artifacts. Imported artifacts that conflict with + // existing resources are ignored. + rpc ImportYumArtifacts(ImportYumArtifactsRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/repositories/*}/yumArtifacts:import" + body: "*" + }; + option (google.longrunning.operation_info) = { + response_type: "google.devtools.artifactregistry.v1.ImportYumArtifactsResponse" + metadata_type: "google.devtools.artifactregistry.v1.ImportYumArtifactsMetadata" + }; + } + + // Lists repositories. + rpc ListRepositories(ListRepositoriesRequest) + returns (ListRepositoriesResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*}/repositories" + }; + option (google.api.method_signature) = "parent"; + } + + // Gets a repository. + rpc GetRepository(GetRepositoryRequest) returns (Repository) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/repositories/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Creates a repository. The returned Operation will finish once the + // repository has been created. Its response will be the created Repository. + rpc CreateRepository(CreateRepositoryRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*}/repositories" + body: "repository" + }; + option (google.api.method_signature) = "parent,repository,repository_id"; + option (google.longrunning.operation_info) = { + response_type: "google.devtools.artifactregistry.v1.Repository" + metadata_type: "google.devtools.artifactregistry.v1.OperationMetadata" + }; + } + + // Updates a repository. + rpc UpdateRepository(UpdateRepositoryRequest) returns (Repository) { + option (google.api.http) = { + patch: "/v1/{repository.name=projects/*/locations/*/repositories/*}" + body: "repository" + }; + option (google.api.method_signature) = "repository,update_mask"; + } + + // Deletes a repository and all of its contents. The returned Operation will + // finish once the repository has been deleted. It will not have any Operation + // metadata and will return a google.protobuf.Empty response. + rpc DeleteRepository(DeleteRepositoryRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/locations/*/repositories/*}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "google.devtools.artifactregistry.v1.OperationMetadata" + }; + } + + // Lists packages. + rpc ListPackages(ListPackagesRequest) returns (ListPackagesResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/repositories/*}/packages" + }; + option (google.api.method_signature) = "parent"; + } + + // Gets a package. + rpc GetPackage(GetPackageRequest) returns (Package) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/repositories/*/packages/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Deletes a package and all of its versions and tags. The returned operation + // will complete once the package has been deleted. + rpc DeletePackage(DeletePackageRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/locations/*/repositories/*/packages/*}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "google.devtools.artifactregistry.v1.OperationMetadata" + }; + } + + // Lists versions. + rpc ListVersions(ListVersionsRequest) returns (ListVersionsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/repositories/*/packages/*}/versions" + }; + option (google.api.method_signature) = "parent"; + } + + // Gets a version + rpc GetVersion(GetVersionRequest) returns (Version) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/repositories/*/packages/*/versions/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Deletes a version and all of its content. The returned operation will + // complete once the version has been deleted. + rpc DeleteVersion(DeleteVersionRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/locations/*/repositories/*/packages/*/versions/*}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "google.devtools.artifactregistry.v1.OperationMetadata" + }; + } + + // Deletes multiple versions across a repository. The returned operation will + // complete once the versions have been deleted. + rpc BatchDeleteVersions(BatchDeleteVersionsRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/repositories/*/packages/*}/versions:batchDelete" + body: "*" + }; + option (google.api.method_signature) = "parent,names"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "google.devtools.artifactregistry.v1.BatchDeleteVersionsMetadata" + }; + } + + // Updates a version. + rpc UpdateVersion(UpdateVersionRequest) returns (Version) { + option (google.api.http) = { + patch: "/v1/{version.name=projects/*/locations/*/repositories/*/packages/*/versions/*}" + body: "version" + }; + option (google.api.method_signature) = "version,update_mask"; + } + + // Lists files. + rpc ListFiles(ListFilesRequest) returns (ListFilesResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/repositories/*}/files" + }; + option (google.api.method_signature) = "parent"; + } + + // Gets a file. + rpc GetFile(GetFileRequest) returns (File) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/repositories/*/files/**}" + }; + option (google.api.method_signature) = "name"; + } + + // Deletes a file and all of its content. It is only allowed on generic + // repositories. The returned operation will complete once the file has been + // deleted. + rpc DeleteFile(DeleteFileRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/locations/*/repositories/*/files/*}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "google.devtools.artifactregistry.v1.OperationMetadata" + }; + } + + // Updates a file. + rpc UpdateFile(UpdateFileRequest) returns (File) { + option (google.api.http) = { + patch: "/v1/{file.name=projects/*/locations/*/repositories/*/files/*}" + body: "file" + }; + option (google.api.method_signature) = "file,update_mask"; + } + + // Lists tags. + rpc ListTags(ListTagsRequest) returns (ListTagsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/repositories/*/packages/*}/tags" + }; + option (google.api.method_signature) = "parent"; + } + + // Gets a tag. + rpc GetTag(GetTagRequest) returns (Tag) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/repositories/*/packages/*/tags/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Creates a tag. + rpc CreateTag(CreateTagRequest) returns (Tag) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/repositories/*/packages/*}/tags" + body: "tag" + }; + option (google.api.method_signature) = "parent,tag,tag_id"; + } + + // Updates a tag. + rpc UpdateTag(UpdateTagRequest) returns (Tag) { + option (google.api.http) = { + patch: "/v1/{tag.name=projects/*/locations/*/repositories/*/packages/*/tags/*}" + body: "tag" + }; + option (google.api.method_signature) = "tag,update_mask"; + } + + // Deletes a tag. + rpc DeleteTag(DeleteTagRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/locations/*/repositories/*/packages/*/tags/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Creates a rule. + rpc CreateRule(CreateRuleRequest) returns (Rule) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/repositories/*}/rules" + body: "rule" + }; + option (google.api.method_signature) = "parent,rule,rule_id"; + } + + // Lists rules. + rpc ListRules(ListRulesRequest) returns (ListRulesResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/repositories/*}/rules" + }; + option (google.api.method_signature) = "parent"; + } + + // Gets a rule. + rpc GetRule(GetRuleRequest) returns (Rule) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/repositories/*/rules/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Updates a rule. + rpc UpdateRule(UpdateRuleRequest) returns (Rule) { + option (google.api.http) = { + patch: "/v1/{rule.name=projects/*/locations/*/repositories/*/rules/*}" + body: "rule" + }; + option (google.api.method_signature) = "rule,update_mask"; + } + + // Deletes a rule. + rpc DeleteRule(DeleteRuleRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/locations/*/repositories/*/rules/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Updates the IAM policy for a given resource. + rpc SetIamPolicy(google.iam.v1.SetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + post: "/v1/{resource=projects/*/locations/*/repositories/*}:setIamPolicy" + body: "*" + }; + } + + // Gets the IAM policy for a given resource. + rpc GetIamPolicy(google.iam.v1.GetIamPolicyRequest) + returns (google.iam.v1.Policy) { + option (google.api.http) = { + get: "/v1/{resource=projects/*/locations/*/repositories/*}:getIamPolicy" + }; + } + + // Tests if the caller has a list of permissions on a resource. + rpc TestIamPermissions(google.iam.v1.TestIamPermissionsRequest) + returns (google.iam.v1.TestIamPermissionsResponse) { + option (google.api.http) = { + post: "/v1/{resource=projects/*/locations/*/repositories/*}:testIamPermissions" + body: "*" + }; + } + + // Retrieves the Settings for the Project. + rpc GetProjectSettings(GetProjectSettingsRequest) returns (ProjectSettings) { + option (google.api.http) = { + get: "/v1/{name=projects/*/projectSettings}" + }; + option (google.api.method_signature) = "name"; + } + + // Updates the Settings for the Project. + rpc UpdateProjectSettings(UpdateProjectSettingsRequest) + returns (ProjectSettings) { + option (google.api.http) = { + patch: "/v1/{project_settings.name=projects/*/projectSettings}" + body: "project_settings" + }; + option (google.api.method_signature) = "project_settings,update_mask"; + } + + // Retrieves the VPCSC Config for the Project. + rpc GetVPCSCConfig(GetVPCSCConfigRequest) returns (VPCSCConfig) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/vpcscConfig}" + }; + option (google.api.method_signature) = "name"; + } + + // Updates the VPCSC Config for the Project. + rpc UpdateVPCSCConfig(UpdateVPCSCConfigRequest) returns (VPCSCConfig) { + option (google.api.http) = { + patch: "/v1/{vpcsc_config.name=projects/*/locations/*/vpcscConfig}" + body: "vpcsc_config" + }; + option (google.api.method_signature) = "vpcsc_config,update_mask"; + } + + // Updates a package. + rpc UpdatePackage(UpdatePackageRequest) returns (Package) { + option (google.api.http) = { + patch: "/v1/{package.name=projects/*/locations/*/repositories/*/packages/*}" + body: "package" + }; + option (google.api.method_signature) = "package,update_mask"; + } + + // Lists attachments. + rpc ListAttachments(ListAttachmentsRequest) + returns (ListAttachmentsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/repositories/*}/attachments" + }; + option (google.api.method_signature) = "parent"; + } + + // Gets an attachment. + rpc GetAttachment(GetAttachmentRequest) returns (Attachment) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/repositories/*/attachments/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Creates an attachment. The returned Operation will finish once the + // attachment has been created. Its response will be the created attachment. + rpc CreateAttachment(CreateAttachmentRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/repositories/*}/attachments" + body: "attachment" + }; + option (google.api.method_signature) = "parent,attachment,attachment_id"; + option (google.longrunning.operation_info) = { + response_type: "google.devtools.artifactregistry.v1.Attachment" + metadata_type: "google.devtools.artifactregistry.v1.OperationMetadata" + }; + } + + // Deletes an attachment. The returned Operation will + // finish once the attachments has been deleted. It will not have any + // Operation metadata and will return a `google.protobuf.Empty` response. + rpc DeleteAttachment(DeleteAttachmentRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/locations/*/repositories/*/attachments/*}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "google.devtools.artifactregistry.v1.OperationMetadata" + }; + } +} + +// Metadata type for longrunning-operations, currently empty. +message OperationMetadata {} diff --git a/proto/google/devtools/artifactregistry/v1/settings.proto b/proto/google/devtools/artifactregistry/v1/settings.proto new file mode 100644 index 0000000..8a2792f --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/settings.proto @@ -0,0 +1,94 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/field_mask.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "SettingsProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// The Artifact Registry settings that apply to a Project. +message ProjectSettings { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/ProjectSettings" + pattern: "projects/{project}/projectSettings" + }; + + // The possible redirection states for legacy repositories. + enum RedirectionState { + // No redirection status has been set. + REDIRECTION_STATE_UNSPECIFIED = 0; + + // Redirection is disabled. + REDIRECTION_FROM_GCR_IO_DISABLED = 1; + + // Redirection is enabled. + REDIRECTION_FROM_GCR_IO_ENABLED = 2; + + // Redirection is enabled, and has been finalized so cannot be reverted. + REDIRECTION_FROM_GCR_IO_FINALIZED = 3 [deprecated = true]; + + // Redirection is enabled and missing images are copied from GCR + REDIRECTION_FROM_GCR_IO_ENABLED_AND_COPYING = 5; + + // Redirection is partially enabled and missing images are copied from GCR + REDIRECTION_FROM_GCR_IO_PARTIAL_AND_COPYING = 6; + } + + // The name of the project's settings. + // + // Always of the form: + // projects/{project-id}/projectSettings + // + // In update request: never set + // In response: always set + string name = 1; + + // The redirection state of the legacy repositories in this project. + RedirectionState legacy_redirection_state = 2; + + // The percentage of pull traffic to redirect from GCR to AR when using + // partial redirection. + int32 pull_percent = 3; +} + +// Gets the redirection status for a project. +message GetProjectSettingsRequest { + // Required. The name of the projectSettings resource. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/ProjectSettings" + } + ]; +} + +// Sets the settings of the project. +message UpdateProjectSettingsRequest { + // The project settings. + ProjectSettings project_settings = 2; + + // Field mask to support partial updates. + google.protobuf.FieldMask update_mask = 3; +} diff --git a/proto/google/devtools/artifactregistry/v1/tag.proto b/proto/google/devtools/artifactregistry/v1/tag.proto new file mode 100644 index 0000000..3708e9a --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/tag.proto @@ -0,0 +1,138 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/resource.proto"; +import "google/protobuf/field_mask.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "TagProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// Tags point to a version and represent an alternative name that can be used +// to access the version. +message Tag { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/Tag" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/packages/{package}/tags/{tag}" + }; + + // The name of the tag, for example: + // "projects/p1/locations/us-central1/repositories/repo1/packages/pkg1/tags/tag1". + // If the package part contains slashes, the slashes are escaped. + // The tag part can only have characters in [a-zA-Z0-9\-._~:@], anything else + // must be URL encoded. + string name = 1; + + // The name of the version the tag refers to, for example: + // `projects/p1/locations/us-central1/repositories/repo1/packages/pkg1/versions/sha256:5243811` + // If the package or version ID parts contain slashes, the slashes are + // escaped. + string version = 2; +} + +// The request to list tags. +message ListTagsRequest { + // The name of the parent package whose tags will be listed. + // For example: + // `projects/p1/locations/us-central1/repositories/repo1/packages/pkg1`. + string parent = 1; + + // An expression for filtering the results of the request. Filter rules are + // case insensitive. The fields eligible for filtering are: + // + // * `name` + // * `version` + // + // Examples of using a filter: + // + // To filter the results of your request to tags with the name `my-tag` in + // package `my-package` in repository `my-repo` in project "`y-project` in + // the us-central region, append the following filter expression to your + // request: + // + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-package/tags/my-tag"` + // + // You can also use wildcards to match any number of characters before or + // after the value: + // + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-package/tags/my*"` + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-package/tags/*tag"` + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-package/tags/*tag*"` + // + // To filter the results of your request to tags applied to the version + // `1.0` in package `my-package`, append the following filter expression to + // your request: + // + // * `version="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-package/versions/1.0"` + string filter = 4; + + // The maximum number of tags to return. Maximum page size is 1,000. + int32 page_size = 2; + + // The next_page_token value returned from a previous list request, if any. + string page_token = 3; +} + +// The response from listing tags. +message ListTagsResponse { + // The tags returned. + repeated Tag tags = 1; + + // The token to retrieve the next page of tags, or empty if there are no + // more tags to return. + string next_page_token = 2; +} + +// The request to retrieve a tag. +message GetTagRequest { + // The name of the tag to retrieve. + string name = 1; +} + +// The request to create a new tag. +message CreateTagRequest { + // The name of the parent resource where the tag will be created. + string parent = 1; + + // The tag id to use for this repository. + string tag_id = 2; + + // The tag to be created. + Tag tag = 3; +} + +// The request to create or update a tag. +message UpdateTagRequest { + // The tag that replaces the resource on the server. + Tag tag = 1; + + // The update mask applies to the resource. For the `FieldMask` definition, + // see + // https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#fieldmask + google.protobuf.FieldMask update_mask = 2; +} + +// The request to delete a tag. +message DeleteTagRequest { + // The name of the tag to delete. + string name = 1; +} diff --git a/proto/google/devtools/artifactregistry/v1/version.proto b/proto/google/devtools/artifactregistry/v1/version.proto new file mode 100644 index 0000000..fc9984d --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/version.proto @@ -0,0 +1,217 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/devtools/artifactregistry/v1/tag.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "VersionProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// The view, which determines what version information is returned in a +// response. +enum VersionView { + // The default / unset value. + // The API will default to the BASIC view. + VERSION_VIEW_UNSPECIFIED = 0; + + // Includes basic information about the version, but not any related tags. + BASIC = 1; + + // Include everything. + FULL = 2; +} + +// The body of a version resource. A version resource represents a +// collection of components, such as files and other data. This may correspond +// to a version in many package management schemes. +message Version { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/Version" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/packages/{package}/versions/{version}" + }; + + // The name of the version, for example: + // `projects/p1/locations/us-central1/repositories/repo1/packages/pkg1/versions/art1`. + // If the package or version ID parts contain slashes, the slashes are + // escaped. + string name = 1; + + // Optional. Description of the version, as specified in its metadata. + string description = 3; + + // The time when the version was created. + google.protobuf.Timestamp create_time = 5; + + // The time when the version was last updated. + google.protobuf.Timestamp update_time = 6; + + // Output only. A list of related tags. Will contain up to 100 tags that + // reference this version. + repeated Tag related_tags = 7; + + // Output only. Repository-specific Metadata stored against this version. + // The fields returned are defined by the underlying repository-specific + // resource. Currently, the resources could be: + // [DockerImage][google.devtools.artifactregistry.v1.DockerImage] + // [MavenArtifact][google.devtools.artifactregistry.v1.MavenArtifact] + google.protobuf.Struct metadata = 8 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. Client specified annotations. + map annotations = 9 [(google.api.field_behavior) = OPTIONAL]; +} + +// The request to list versions. +message ListVersionsRequest { + // The name of the parent resource whose versions will be listed. + string parent = 1; + + // The maximum number of versions to return. Maximum page size is 1,000. + int32 page_size = 2; + + // The next_page_token value returned from a previous list request, if any. + string page_token = 3; + + // The view that should be returned in the response. + VersionView view = 4; + + // Optional. The field to order the results by. + string order_by = 5 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. An expression for filtering the results of the request. Filter + // rules are case insensitive. The fields eligible for filtering are: + // + // * `name` + // * `annotations` + // + // Examples of using a filter: + // + // To filter the results of your request to versions with the name + // `my-version` in project `my-project` in the `us-central` region, in + // repository `my-repo`, append the following filter expression to your + // request: + // + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-package/versions/my-version"` + // + // You can also use wildcards to match any number of characters before or + // after the value: + // + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-package/versions/*version"` + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-package/versions/my*"` + // * `name="projects/my-project/locations/us-central1/repositories/my-repo/packages/my-package/versions/*version*"` + // + // To filter the results of your request to versions with the annotation + // key-value pair [`external_link`: `external_link_value`], append the + // following filter expression to your request: + // + // * `"annotations.external_link:external_link_value"` + // + // To filter just for a specific annotation key `external_link`, append the + // following filter expression to your request: + // + // * `"annotations.external_link"` + // + // If the annotation key or value contains special characters, you can escape + // them by surrounding the value with backticks. For example, to filter the + // results of your request to versions with the annotation key-value pair + // [`external.link`:`https://example.com/my-version`], append the following + // filter expression to your request: + // + // * `` "annotations.`external.link`:`https://example.com/my-version`" `` + // + // You can also filter with annotations with a wildcard to + // match any number of characters before or after the value: + // + // * `` "annotations.*_link:`*example.com*`" `` + string filter = 6 [(google.api.field_behavior) = OPTIONAL]; +} + +// The response from listing versions. +message ListVersionsResponse { + // The versions returned. + repeated Version versions = 1; + + // The token to retrieve the next page of versions, or empty if there are no + // more versions to return. + string next_page_token = 2; +} + +// The request to retrieve a version. +message GetVersionRequest { + // The name of the version to retrieve. + string name = 1; + + // The view that should be returned in the response. + VersionView view = 2; +} + +// The request to delete a version. +message DeleteVersionRequest { + // The name of the version to delete. + string name = 1; + + // By default, a version that is tagged may not be deleted. If force=true, the + // version and any tags pointing to the version are deleted. + bool force = 2; +} + +// The request to delete multiple versions across a repository. +message BatchDeleteVersionsRequest { + // The name of the repository holding all requested versions. + string parent = 1 [(google.api.resource_reference) = { + child_type: "artifactregistry.googleapis.com/Version" + }]; + + // Required. The names of the versions to delete. + // A maximum of 10000 versions can be deleted in a batch. + repeated string names = 2 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/Version" + } + ]; + + // If true, the request is performed without deleting data, following AIP-163. + bool validate_only = 3; +} + +// The metadata of an LRO from deleting multiple versions. +message BatchDeleteVersionsMetadata { + // The versions the operation failed to delete. + repeated string failed_versions = 2; +} + +// The request to update a version. +message UpdateVersionRequest { + // Required. The Version that replaces the resource on the server. + Version version = 1 [(google.api.field_behavior) = REQUIRED]; + + // The update mask applies to the resource. For the `FieldMask` definition, + // see + // https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#fieldmask + google.protobuf.FieldMask update_mask = 2; +} diff --git a/proto/google/devtools/artifactregistry/v1/vpcsc_config.proto b/proto/google/devtools/artifactregistry/v1/vpcsc_config.proto new file mode 100644 index 0000000..00e2aa7 --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/vpcsc_config.proto @@ -0,0 +1,86 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/field_mask.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "VPCSCConfigProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// The Artifact Registry VPC SC config that apply to a Project. +message VPCSCConfig { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/VpcscConfig" + pattern: "projects/{project}/locations/{location}/vpcscConfig" + }; + + // VPCSCPolicy is the VPC SC policy for project and location. + enum VPCSCPolicy { + // VPCSC_POLICY_UNSPECIFIED - the VPS SC policy is not defined. + // When VPS SC policy is not defined - the Service will use the default + // behavior (VPCSC_DENY). + VPCSC_POLICY_UNSPECIFIED = 0; + + // VPCSC_DENY - repository will block the requests to the Upstreams for the + // Remote Repositories if the resource is in the perimeter. + DENY = 1; + + // VPCSC_ALLOW - repository will allow the requests to the Upstreams for the + // Remote Repositories if the resource is in the perimeter. + ALLOW = 2; + } + + // The name of the project's VPC SC Config. + // + // Always of the form: + // projects/{projectID}/locations/{location}/vpcscConfig + // + // In update request: never set + // In response: always set + string name = 1; + + // The project per location VPC SC policy that defines the VPC SC behavior for + // the Remote Repository (Allow/Deny). + VPCSCPolicy vpcsc_policy = 2; +} + +// Gets the VPC SC config for a project. +message GetVPCSCConfigRequest { + // Required. The name of the VPCSCConfig resource. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "artifactregistry.googleapis.com/VpcscConfig" + } + ]; +} + +// Sets the VPCSC config of the project. +message UpdateVPCSCConfigRequest { + // The project config. + VPCSCConfig vpcsc_config = 1; + + // Field mask to support partial updates. + google.protobuf.FieldMask update_mask = 2; +} diff --git a/proto/google/devtools/artifactregistry/v1/yum_artifact.proto b/proto/google/devtools/artifactregistry/v1/yum_artifact.proto new file mode 100644 index 0000000..5af5803 --- /dev/null +++ b/proto/google/devtools/artifactregistry/v1/yum_artifact.proto @@ -0,0 +1,106 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.devtools.artifactregistry.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/rpc/status.proto"; + +option csharp_namespace = "Google.Cloud.ArtifactRegistry.V1"; +option go_package = "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb;artifactregistrypb"; +option java_multiple_files = true; +option java_outer_classname = "YumArtifactProto"; +option java_package = "com.google.devtools.artifactregistry.v1"; +option php_namespace = "Google\\Cloud\\ArtifactRegistry\\V1"; +option ruby_package = "Google::Cloud::ArtifactRegistry::V1"; + +// A detailed representation of a Yum artifact. +message YumArtifact { + option (google.api.resource) = { + type: "artifactregistry.googleapis.com/YumArtifact" + pattern: "projects/{project}/locations/{location}/repositories/{repository}/yumArtifacts/{yum_artifact}" + }; + + // Package type is either binary or source. + enum PackageType { + // Package type is not specified. + PACKAGE_TYPE_UNSPECIFIED = 0; + + // Binary package (.rpm). + BINARY = 1; + + // Source package (.srpm). + SOURCE = 2; + } + + // Output only. The Artifact Registry resource name of the artifact. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. The yum package name of the artifact. + string package_name = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. An artifact is a binary or source package. + PackageType package_type = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Operating system architecture of the artifact. + string architecture = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Google Cloud Storage location where the artifacts currently reside. +message ImportYumArtifactsGcsSource { + // Cloud Storage paths URI (e.g., gs://my_bucket//my_object). + repeated string uris = 1; + + // Supports URI wildcards for matching multiple objects from a single URI. + bool use_wildcards = 2; +} + +// The request to import new yum artifacts. +message ImportYumArtifactsRequest { + // The source location of the package binaries. + oneof source { + // Google Cloud Storage location where input content is located. + ImportYumArtifactsGcsSource gcs_source = 2; + } + + // The name of the parent resource where the artifacts will be imported. + string parent = 1; +} + +// Error information explaining why a package was not imported. +message ImportYumArtifactsErrorInfo { + // The source that was not imported. + oneof source { + // Google Cloud Storage location requested. + ImportYumArtifactsGcsSource gcs_source = 1; + } + + // The detailed error status. + google.rpc.Status error = 2; +} + +// The response message from importing YUM artifacts. +message ImportYumArtifactsResponse { + // The yum artifacts imported. + repeated YumArtifact yum_artifacts = 1; + + // Detailed error info for packages that were not imported. + repeated ImportYumArtifactsErrorInfo errors = 2; +} + +// The operation metadata for importing artifacts. +message ImportYumArtifactsMetadata {} diff --git a/proto/google/iam/v1/BUILD b/proto/google/iam/v1/BUILD new file mode 100644 index 0000000..8c23663 --- /dev/null +++ b/proto/google/iam/v1/BUILD @@ -0,0 +1,30 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "iam_policy_proto", + srcs = ["iam_policy.proto"], + strip_import_prefix = "/proto/", + deps = [ + ":options_proto", + ":policy_proto", + "//proto/google:com_google_protobuf", + "//proto/google:googleapis", + ], +) + +proto_library( + name = "policy_proto", + srcs = ["policy.proto"], + strip_import_prefix = "/proto/", + deps = [ + "//proto/google/type:expr_proto", + ], +) + +proto_library( + name = "options_proto", + srcs = ["options.proto"], + strip_import_prefix = "/proto/", +) diff --git a/proto/google/iam/v1/iam_policy.proto b/proto/google/iam/v1/iam_policy.proto new file mode 100644 index 0000000..402b345 --- /dev/null +++ b/proto/google/iam/v1/iam_policy.proto @@ -0,0 +1,155 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.iam.v1; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/iam/v1/options.proto"; +import "google/iam/v1/policy.proto"; +import "google/protobuf/field_mask.proto"; + +option cc_enable_arenas = true; +option csharp_namespace = "Google.Cloud.Iam.V1"; +option go_package = "google.golang.org/genproto/googleapis/iam/v1;iam"; +option java_multiple_files = true; +option java_outer_classname = "IamPolicyProto"; +option java_package = "com.google.iam.v1"; +option php_namespace = "Google\\Cloud\\Iam\\V1"; + +// API Overview +// +// +// Manages Identity and Access Management (IAM) policies. +// +// Any implementation of an API that offers access control features +// implements the google.iam.v1.IAMPolicy interface. +// +// ## Data model +// +// Access control is applied when a principal (user or service account), takes +// some action on a resource exposed by a service. Resources, identified by +// URI-like names, are the unit of access control specification. Service +// implementations can choose the granularity of access control and the +// supported permissions for their resources. +// For example one database service may allow access control to be +// specified only at the Table level, whereas another might allow access control +// to also be specified at the Column level. +// +// ## Policy Structure +// +// See google.iam.v1.Policy +// +// This is intentionally not a CRUD style API because access control policies +// are created and deleted implicitly with the resources to which they are +// attached. +service IAMPolicy { + option (google.api.default_host) = "iam-meta-api.googleapis.com"; + + // Sets the access control policy on the specified resource. Replaces any + // existing policy. + // + // Can return `NOT_FOUND`, `INVALID_ARGUMENT`, and `PERMISSION_DENIED` errors. + rpc SetIamPolicy(SetIamPolicyRequest) returns (Policy) { + option (google.api.http) = { + post: "/v1/{resource=**}:setIamPolicy" + body: "*" + }; + } + + // Gets the access control policy for a resource. + // Returns an empty policy if the resource exists and does not have a policy + // set. + rpc GetIamPolicy(GetIamPolicyRequest) returns (Policy) { + option (google.api.http) = { + post: "/v1/{resource=**}:getIamPolicy" + body: "*" + }; + } + + // Returns permissions that a caller has on the specified resource. + // If the resource does not exist, this will return an empty set of + // permissions, not a `NOT_FOUND` error. + // + // Note: This operation is designed to be used for building permission-aware + // UIs and command-line tools, not for authorization checking. This operation + // may "fail open" without warning. + rpc TestIamPermissions(TestIamPermissionsRequest) returns (TestIamPermissionsResponse) { + option (google.api.http) = { + post: "/v1/{resource=**}:testIamPermissions" + body: "*" + }; + } +} + +// Request message for `SetIamPolicy` method. +message SetIamPolicyRequest { + // REQUIRED: The resource for which the policy is being specified. + // See the operation documentation for the appropriate value for this field. + string resource = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference).type = "*"]; + + // REQUIRED: The complete policy to be applied to the `resource`. The size of + // the policy is limited to a few 10s of KB. An empty policy is a + // valid policy but certain Cloud Platform services (such as Projects) + // might reject them. + Policy policy = 2 [(google.api.field_behavior) = REQUIRED]; + + // OPTIONAL: A FieldMask specifying which fields of the policy to modify. Only + // the fields in the mask will be modified. If no mask is provided, the + // following default mask is used: + // + // `paths: "bindings, etag"` + google.protobuf.FieldMask update_mask = 3; +} + +// Request message for `GetIamPolicy` method. +message GetIamPolicyRequest { + // REQUIRED: The resource for which the policy is being requested. + // See the operation documentation for the appropriate value for this field. + string resource = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference).type = "*"]; + + // OPTIONAL: A `GetPolicyOptions` object for specifying options to + // `GetIamPolicy`. + GetPolicyOptions options = 2; +} + +// Request message for `TestIamPermissions` method. +message TestIamPermissionsRequest { + // REQUIRED: The resource for which the policy detail is being requested. + // See the operation documentation for the appropriate value for this field. + string resource = 1[ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference).type = "*"]; + + // The set of permissions to check for the `resource`. Permissions with + // wildcards (such as '*' or 'storage.*') are not allowed. For more + // information see + // [IAM Overview](https://cloud.google.com/iam/docs/overview#permissions). + repeated string permissions = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// Response message for `TestIamPermissions` method. +message TestIamPermissionsResponse { + // A subset of `TestPermissionsRequest.permissions` that the caller is + // allowed. + repeated string permissions = 1; +} diff --git a/proto/google/iam/v1/options.proto b/proto/google/iam/v1/options.proto new file mode 100644 index 0000000..4c94a36 --- /dev/null +++ b/proto/google/iam/v1/options.proto @@ -0,0 +1,48 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.iam.v1; + +option cc_enable_arenas = true; +option csharp_namespace = "Google.Cloud.Iam.V1"; +option go_package = "google.golang.org/genproto/googleapis/iam/v1;iam"; +option java_multiple_files = true; +option java_outer_classname = "OptionsProto"; +option java_package = "com.google.iam.v1"; +option php_namespace = "Google\\Cloud\\Iam\\V1"; + +// Encapsulates settings provided to GetIamPolicy. +message GetPolicyOptions { + // Optional. The maximum policy version that will be used to format the + // policy. + // + // Valid values are 0, 1, and 3. Requests specifying an invalid value will be + // rejected. + // + // Requests for policies with any conditional role bindings must specify + // version 3. Policies with no conditional role bindings may specify any valid + // value or leave the field unset. + // + // The policy in the response might use the policy version that you specified, + // or it might use a lower policy version. For example, if you specify version + // 3, but the policy has no conditional role bindings, the response uses + // version 1. + // + // To learn which resources support conditions in their IAM policies, see the + // [IAM + // documentation](https://cloud.google.com/iam/help/conditions/resource-policies). + int32 requested_policy_version = 1; +} diff --git a/proto/google/iam/v1/policy.proto b/proto/google/iam/v1/policy.proto new file mode 100644 index 0000000..347ae78 --- /dev/null +++ b/proto/google/iam/v1/policy.proto @@ -0,0 +1,402 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.iam.v1; + +import "google/type/expr.proto"; + +option cc_enable_arenas = true; +option csharp_namespace = "Google.Cloud.Iam.V1"; +option go_package = "google.golang.org/genproto/googleapis/iam/v1;iam"; +option java_multiple_files = true; +option java_outer_classname = "PolicyProto"; +option java_package = "com.google.iam.v1"; +option php_namespace = "Google\\Cloud\\Iam\\V1"; + +// An Identity and Access Management (IAM) policy, which specifies access +// controls for Google Cloud resources. +// +// +// A `Policy` is a collection of `bindings`. A `binding` binds one or more +// `members`, or principals, to a single `role`. Principals can be user +// accounts, service accounts, Google groups, and domains (such as G Suite). A +// `role` is a named list of permissions; each `role` can be an IAM predefined +// role or a user-created custom role. +// +// For some types of Google Cloud resources, a `binding` can also specify a +// `condition`, which is a logical expression that allows access to a resource +// only if the expression evaluates to `true`. A condition can add constraints +// based on attributes of the request, the resource, or both. To learn which +// resources support conditions in their IAM policies, see the +// [IAM documentation](https://cloud.google.com/iam/help/conditions/resource-policies). +// +// **JSON example:** +// +// { +// "bindings": [ +// { +// "role": "roles/resourcemanager.organizationAdmin", +// "members": [ +// "user:mike@example.com", +// "group:admins@example.com", +// "domain:google.com", +// "serviceAccount:my-project-id@appspot.gserviceaccount.com" +// ] +// }, +// { +// "role": "roles/resourcemanager.organizationViewer", +// "members": [ +// "user:eve@example.com" +// ], +// "condition": { +// "title": "expirable access", +// "description": "Does not grant access after Sep 2020", +// "expression": "request.time < timestamp('2020-10-01T00:00:00.000Z')", +// } +// } +// ], +// "etag": "BwWWja0YfJA=", +// "version": 3 +// } +// +// **YAML example:** +// +// bindings: +// - members: +// - user:mike@example.com +// - group:admins@example.com +// - domain:google.com +// - serviceAccount:my-project-id@appspot.gserviceaccount.com +// role: roles/resourcemanager.organizationAdmin +// - members: +// - user:eve@example.com +// role: roles/resourcemanager.organizationViewer +// condition: +// title: expirable access +// description: Does not grant access after Sep 2020 +// expression: request.time < timestamp('2020-10-01T00:00:00.000Z') +// etag: BwWWja0YfJA= +// version: 3 +// +// For a description of IAM and its features, see the +// [IAM documentation](https://cloud.google.com/iam/docs/). +message Policy { + // Specifies the format of the policy. + // + // Valid values are `0`, `1`, and `3`. Requests that specify an invalid value + // are rejected. + // + // Any operation that affects conditional role bindings must specify version + // `3`. This requirement applies to the following operations: + // + // * Getting a policy that includes a conditional role binding + // * Adding a conditional role binding to a policy + // * Changing a conditional role binding in a policy + // * Removing any role binding, with or without a condition, from a policy + // that includes conditions + // + // **Important:** If you use IAM Conditions, you must include the `etag` field + // whenever you call `setIamPolicy`. If you omit this field, then IAM allows + // you to overwrite a version `3` policy with a version `1` policy, and all of + // the conditions in the version `3` policy are lost. + // + // If a policy does not include any conditions, operations on that policy may + // specify any valid version or leave the field unset. + // + // To learn which resources support conditions in their IAM policies, see the + // [IAM documentation](https://cloud.google.com/iam/help/conditions/resource-policies). + int32 version = 1; + + // Associates a list of `members`, or principals, with a `role`. Optionally, + // may specify a `condition` that determines how and when the `bindings` are + // applied. Each of the `bindings` must contain at least one principal. + // + // The `bindings` in a `Policy` can refer to up to 1,500 principals; up to 250 + // of these principals can be Google groups. Each occurrence of a principal + // counts towards these limits. For example, if the `bindings` grant 50 + // different roles to `user:alice@example.com`, and not to any other + // principal, then you can add another 1,450 principals to the `bindings` in + // the `Policy`. + repeated Binding bindings = 4; + + // Specifies cloud audit logging configuration for this policy. + repeated AuditConfig audit_configs = 6; + + // `etag` is used for optimistic concurrency control as a way to help + // prevent simultaneous updates of a policy from overwriting each other. + // It is strongly suggested that systems make use of the `etag` in the + // read-modify-write cycle to perform policy updates in order to avoid race + // conditions: An `etag` is returned in the response to `getIamPolicy`, and + // systems are expected to put that etag in the request to `setIamPolicy` to + // ensure that their change will be applied to the same version of the policy. + // + // **Important:** If you use IAM Conditions, you must include the `etag` field + // whenever you call `setIamPolicy`. If you omit this field, then IAM allows + // you to overwrite a version `3` policy with a version `1` policy, and all of + // the conditions in the version `3` policy are lost. + bytes etag = 3; +} + +// Associates `members`, or principals, with a `role`. +message Binding { + // Role that is assigned to the list of `members`, or principals. + // For example, `roles/viewer`, `roles/editor`, or `roles/owner`. + string role = 1; + + // Specifies the principals requesting access for a Cloud Platform resource. + // `members` can have the following values: + // + // * `allUsers`: A special identifier that represents anyone who is + // on the internet; with or without a Google account. + // + // * `allAuthenticatedUsers`: A special identifier that represents anyone + // who is authenticated with a Google account or a service account. + // + // * `user:{emailid}`: An email address that represents a specific Google + // account. For example, `alice@example.com` . + // + // + // * `serviceAccount:{emailid}`: An email address that represents a service + // account. For example, `my-other-app@appspot.gserviceaccount.com`. + // + // * `group:{emailid}`: An email address that represents a Google group. + // For example, `admins@example.com`. + // + // * `deleted:user:{emailid}?uid={uniqueid}`: An email address (plus unique + // identifier) representing a user that has been recently deleted. For + // example, `alice@example.com?uid=123456789012345678901`. If the user is + // recovered, this value reverts to `user:{emailid}` and the recovered user + // retains the role in the binding. + // + // * `deleted:serviceAccount:{emailid}?uid={uniqueid}`: An email address (plus + // unique identifier) representing a service account that has been recently + // deleted. For example, + // `my-other-app@appspot.gserviceaccount.com?uid=123456789012345678901`. + // If the service account is undeleted, this value reverts to + // `serviceAccount:{emailid}` and the undeleted service account retains the + // role in the binding. + // + // * `deleted:group:{emailid}?uid={uniqueid}`: An email address (plus unique + // identifier) representing a Google group that has been recently + // deleted. For example, `admins@example.com?uid=123456789012345678901`. If + // the group is recovered, this value reverts to `group:{emailid}` and the + // recovered group retains the role in the binding. + // + // + // * `domain:{domain}`: The G Suite domain (primary) that represents all the + // users of that domain. For example, `google.com` or `example.com`. + // + // + repeated string members = 2; + + // The condition that is associated with this binding. + // + // If the condition evaluates to `true`, then this binding applies to the + // current request. + // + // If the condition evaluates to `false`, then this binding does not apply to + // the current request. However, a different role binding might grant the same + // role to one or more of the principals in this binding. + // + // To learn which resources support conditions in their IAM policies, see the + // [IAM + // documentation](https://cloud.google.com/iam/help/conditions/resource-policies). + google.type.Expr condition = 3; +} + +// Specifies the audit configuration for a service. +// The configuration determines which permission types are logged, and what +// identities, if any, are exempted from logging. +// An AuditConfig must have one or more AuditLogConfigs. +// +// If there are AuditConfigs for both `allServices` and a specific service, +// the union of the two AuditConfigs is used for that service: the log_types +// specified in each AuditConfig are enabled, and the exempted_members in each +// AuditLogConfig are exempted. +// +// Example Policy with multiple AuditConfigs: +// +// { +// "audit_configs": [ +// { +// "service": "allServices", +// "audit_log_configs": [ +// { +// "log_type": "DATA_READ", +// "exempted_members": [ +// "user:jose@example.com" +// ] +// }, +// { +// "log_type": "DATA_WRITE" +// }, +// { +// "log_type": "ADMIN_READ" +// } +// ] +// }, +// { +// "service": "sampleservice.googleapis.com", +// "audit_log_configs": [ +// { +// "log_type": "DATA_READ" +// }, +// { +// "log_type": "DATA_WRITE", +// "exempted_members": [ +// "user:aliya@example.com" +// ] +// } +// ] +// } +// ] +// } +// +// For sampleservice, this policy enables DATA_READ, DATA_WRITE and ADMIN_READ +// logging. It also exempts jose@example.com from DATA_READ logging, and +// aliya@example.com from DATA_WRITE logging. +message AuditConfig { + // Specifies a service that will be enabled for audit logging. + // For example, `storage.googleapis.com`, `cloudsql.googleapis.com`. + // `allServices` is a special value that covers all services. + string service = 1; + + // The configuration for logging of each type of permission. + repeated AuditLogConfig audit_log_configs = 3; +} + +// Provides the configuration for logging a type of permissions. +// Example: +// +// { +// "audit_log_configs": [ +// { +// "log_type": "DATA_READ", +// "exempted_members": [ +// "user:jose@example.com" +// ] +// }, +// { +// "log_type": "DATA_WRITE" +// } +// ] +// } +// +// This enables 'DATA_READ' and 'DATA_WRITE' logging, while exempting +// jose@example.com from DATA_READ logging. +message AuditLogConfig { + // The list of valid permission types for which logging can be configured. + // Admin writes are always logged, and are not configurable. + enum LogType { + // Default case. Should never be this. + LOG_TYPE_UNSPECIFIED = 0; + + // Admin reads. Example: CloudIAM getIamPolicy + ADMIN_READ = 1; + + // Data writes. Example: CloudSQL Users create + DATA_WRITE = 2; + + // Data reads. Example: CloudSQL Users list + DATA_READ = 3; + } + + // The log type that this config enables. + LogType log_type = 1; + + // Specifies the identities that do not cause logging for this type of + // permission. + // Follows the same format of [Binding.members][google.iam.v1.Binding.members]. + repeated string exempted_members = 2; +} + +// The difference delta between two policies. +message PolicyDelta { + // The delta for Bindings between two policies. + repeated BindingDelta binding_deltas = 1; + + // The delta for AuditConfigs between two policies. + repeated AuditConfigDelta audit_config_deltas = 2; +} + +// One delta entry for Binding. Each individual change (only one member in each +// entry) to a binding will be a separate entry. +message BindingDelta { + // The type of action performed on a Binding in a policy. + enum Action { + // Unspecified. + ACTION_UNSPECIFIED = 0; + + // Addition of a Binding. + ADD = 1; + + // Removal of a Binding. + REMOVE = 2; + } + + // The action that was performed on a Binding. + // Required + Action action = 1; + + // Role that is assigned to `members`. + // For example, `roles/viewer`, `roles/editor`, or `roles/owner`. + // Required + string role = 2; + + // A single identity requesting access for a Cloud Platform resource. + // Follows the same format of Binding.members. + // Required + string member = 3; + + // The condition that is associated with this binding. + google.type.Expr condition = 4; +} + +// One delta entry for AuditConfig. Each individual change (only one +// exempted_member in each entry) to a AuditConfig will be a separate entry. +message AuditConfigDelta { + // The type of action performed on an audit configuration in a policy. + enum Action { + // Unspecified. + ACTION_UNSPECIFIED = 0; + + // Addition of an audit configuration. + ADD = 1; + + // Removal of an audit configuration. + REMOVE = 2; + } + + // The action that was performed on an audit configuration in a policy. + // Required + Action action = 1; + + // Specifies a service that was configured for Cloud Audit Logging. + // For example, `storage.googleapis.com`, `cloudsql.googleapis.com`. + // `allServices` is a special value that covers all services. + // Required + string service = 2; + + // A single identity that is exempted from "data access" audit + // logging for the `service` specified above. + // Follows the same format of Binding.members. + string exempted_member = 3; + + // Specifies the log_type that was be enabled. ADMIN_ACTIVITY is always + // enabled, and cannot be configured. + // Required + string log_type = 4; +} diff --git a/proto/google/longrunning/BUILD b/proto/google/longrunning/BUILD new file mode 100644 index 0000000..b2d2adb --- /dev/null +++ b/proto/google/longrunning/BUILD @@ -0,0 +1,14 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "operations_proto", + srcs = ["operations.proto"], + strip_import_prefix = "/proto/", + deps = [ + "//proto/google:com_google_protobuf", + "//proto/google:google_rpc", + "//proto/google:googleapis", + ], +) diff --git a/proto/google/longrunning/README.md b/proto/google/longrunning/README.md new file mode 100644 index 0000000..7eba891 --- /dev/null +++ b/proto/google/longrunning/README.md @@ -0,0 +1,31 @@ +## Long-running Operations API + +This package contains the definition of an abstract interface that +manages long running operations with API services. + +### Operation + +The primary message to understand within LRO is the `Operation` message. +Operations have a unique name (in the context of a particular endpoint). +Additionally, a service (called `Operations` -- plural) defines the interface +for querying the state of any given operation. + +APIs that implement a concept of long-running operations are encouraged +to follow this pattern: When a caller invokes an API backend to start a job... + + * The API backend starts asychronous work to fulfill the caller's + request, and generates a unique name (the `Operation` name) to refer + to the ongoing asychronous work. + * The API backend immediately returns the `Operation` back to the caller. + * The caller can invoke the API methods defined in the `Operations` service + to get the current status of the asychronous work, and also to + discover the final result (success or error). + +For Google APIs, the implementation of this pattern and the use of this +proto are part of our [design rules][operations-rules]. Additionally, our +[API client tooling][gapic-generator] seeks to be intelligent about these, to +improve the client API consumption experience. Therefore, APIs outside of +Google can also benefit by following this same pattern. + + [operations-rules]: https://cloud.google.com/apis/design/design_patterns#long_running_operations + [gapic-generator]: https://github.com/googleapis/gapic-generator diff --git a/proto/google/longrunning/longrunning.yaml b/proto/google/longrunning/longrunning.yaml new file mode 100644 index 0000000..fd73549 --- /dev/null +++ b/proto/google/longrunning/longrunning.yaml @@ -0,0 +1,78 @@ +type: google.api.Service +config_version: 3 +name: longrunning.googleapis.com +title: Long Running Operations API + +apis: +- name: google.longrunning.Operations + +types: +- name: google.longrunning.OperationInfo + +documentation: + overview: |- + # Long Running Operation API + + This package contains the definition of Long Running Operation (LRO) + interface. It is a standard interface that API services can implement for + managing asynchronous operations. + + + ## What are Long Running Operations? + + A Long Running Operation (LRO) is a way of representing an action that may + take a significant amount of time to complete. For example, an API call + that starts exporting a large amount of data could take quite a while to + complete and is therefore best represented as an LRO. A common rule of + thumb is to think of LROs as "API promises" that represent the result of + some ongoing action. + + For more information, see [AIP-151](https://google.aip.dev/151). + + ## Using LROs + + If an API method could potentially take long time to finish, the method + should return a long running operation instead of a direct response. This + means that even if there are situations where the response could be + immediate, the API should still return an LRO -- it just may be already + marked as completed. + For example, if a data export operation is called on an empty resource, + the operation itself may be possible to execute immediately, and would + result in an already completed LRO. + + Additionally, the operation should be managed using the LRO interface, + which allows clients to poll the operation for status updates or cancel it + entirely. + + Finally, an LRO represents an action and as a result, the operation is not + created directly. Instead, the operation comes into existence as a + side-effect of the action it represents. For example, an RPC called + `ExportData` would + create and return an LRO. This means that there should never be an RPC + called `CreateOperation`. + + This also means that any permissions on the operation would be based on + the action it represents. Any immediate side effects of starting the + operation must be visible in the service as soon as the LRO is returned. + For example, if an LRO is returned when creating a resource, that resource + should be visible in the API immediately, but be in a non-final state + until the LRO is completed. + + ## LROs versus Jobs + + A [job](https://google.aip.dev/152) is a common design pattern often used + in data processing that tends to be used to represent some contained piece + of work that would be stored, re-run, and modified over time. Jobs also + typically + interact with multiple resources and are created, deleted, and updated + directly as independent resources. + + Jobs can also offer support for more complex actions such as pausing and + resuming an individual job, where each action could return an LRO as a + response. + + In general, if an action may take a while but it represents a single piece + of work, it's best to represent the response as an LRO. If the action is + something more complex (for example, it involves lots of resources and + can't be created as a byproduct of a single action), it may make more + sense to represent it as a job. diff --git a/proto/google/longrunning/longrunning_gapic.yaml b/proto/google/longrunning/longrunning_gapic.yaml new file mode 100644 index 0000000..d6502cf --- /dev/null +++ b/proto/google/longrunning/longrunning_gapic.yaml @@ -0,0 +1,54 @@ +type: com.google.api.codegen.ConfigProto +config_schema_version: 2.0.0 +language_settings: + java: + package_name: com.google.longrunning + python: + package_name: google.longrunning.gapic + go: + package_name: cloud.google.com/go/longrunning/autogen + domain_layer_location: cloud.google.com/go/longrunning + csharp: + package_name: Google.LongRunning + ruby: + package_name: Google::Longrunning + php: + package_name: Google\LongRunning + nodejs: + package_name: longrunning +interfaces: +- name: google.longrunning.Operations + required_constructor_params: + - service_address + - scopes + retry_params_def: + - name: default + initial_retry_delay_millis: 100 + retry_delay_multiplier: 1.3 + max_retry_delay_millis: 60000 + initial_rpc_timeout_millis: 90000 + rpc_timeout_multiplier: 1 + max_rpc_timeout_millis: 90000 + total_timeout_millis: 600000 + methods: + - name: GetOperation + retry_params_name: default + - name: ListOperations + retry_params_name: default + - name: CancelOperation + retry_codes_name: idempotent + retry_params_name: default + - name: DeleteOperation + retry_codes_name: idempotent + retry_params_name: default + - name: WaitOperation + surface_treatments: + - include_languages: + - go + - java + - csharp + - ruby + - nodejs + - python + - php + visibility: DISABLED diff --git a/proto/google/longrunning/longrunning_grpc_service_config.json b/proto/google/longrunning/longrunning_grpc_service_config.json new file mode 100644 index 0000000..a5a9ae9 --- /dev/null +++ b/proto/google/longrunning/longrunning_grpc_service_config.json @@ -0,0 +1,34 @@ +{ + "methodConfig": [ + { + "name": [ + { + "service": "google.longrunning.Operations", + "method": "GetOperation" + }, + { + "service": "google.longrunning.Operations", + "method": "ListOperations" + }, + { + "service": "google.longrunning.Operations", + "method": "CancelOperation" + }, + { + "service": "google.longrunning.Operations", + "method": "DeleteOperation" + } + ], + "timeout": "10s", + "retryPolicy": { + "maxAttempts": 5, + "initialBackoff": "0.5s", + "maxBackoff": "10s", + "backoffMultiplier": 2.0, + "retryableStatusCodes": [ + "UNAVAILABLE" + ] + } + } + ] +} diff --git a/proto/google/longrunning/operations.proto b/proto/google/longrunning/operations.proto new file mode 100644 index 0000000..bb42620 --- /dev/null +++ b/proto/google/longrunning/operations.proto @@ -0,0 +1,246 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.longrunning; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/descriptor.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/empty.proto"; +import "google/rpc/status.proto"; + +option cc_enable_arenas = true; +option csharp_namespace = "Google.LongRunning"; +option go_package = "cloud.google.com/go/longrunning/autogen/longrunningpb;longrunningpb"; +option java_multiple_files = true; +option java_outer_classname = "OperationsProto"; +option java_package = "com.google.longrunning"; +option objc_class_prefix = "GLRUN"; +option php_namespace = "Google\\LongRunning"; + +extend google.protobuf.MethodOptions { + // Additional information regarding long-running operations. + // In particular, this specifies the types that are returned from + // long-running operations. + // + // Required for methods that return `google.longrunning.Operation`; invalid + // otherwise. + google.longrunning.OperationInfo operation_info = 1049; +} + +// Manages long-running operations with an API service. +// +// When an API method normally takes long time to complete, it can be designed +// to return [Operation][google.longrunning.Operation] to the client, and the +// client can use this interface to receive the real response asynchronously by +// polling the operation resource, or pass the operation resource to another API +// (such as Pub/Sub API) to receive the response. Any API service that returns +// long-running operations should implement the `Operations` interface so +// developers can have a consistent client experience. +service Operations { + option (google.api.default_host) = "longrunning.googleapis.com"; + + // Lists operations that match the specified filter in the request. If the + // server doesn't support this method, it returns `UNIMPLEMENTED`. + rpc ListOperations(ListOperationsRequest) returns (ListOperationsResponse) { + option (google.api.http) = { + get: "/v1/{name=operations}" + }; + option (google.api.method_signature) = "name,filter"; + } + + // Gets the latest state of a long-running operation. Clients can use this + // method to poll the operation result at intervals as recommended by the API + // service. + rpc GetOperation(GetOperationRequest) returns (Operation) { + option (google.api.http) = { + get: "/v1/{name=operations/**}" + }; + option (google.api.method_signature) = "name"; + } + + // Deletes a long-running operation. This method indicates that the client is + // no longer interested in the operation result. It does not cancel the + // operation. If the server doesn't support this method, it returns + // `google.rpc.Code.UNIMPLEMENTED`. + rpc DeleteOperation(DeleteOperationRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{name=operations/**}" + }; + option (google.api.method_signature) = "name"; + } + + // Starts asynchronous cancellation on a long-running operation. The server + // makes a best effort to cancel the operation, but success is not + // guaranteed. If the server doesn't support this method, it returns + // `google.rpc.Code.UNIMPLEMENTED`. Clients can use + // [Operations.GetOperation][google.longrunning.Operations.GetOperation] or + // other methods to check whether the cancellation succeeded or whether the + // operation completed despite cancellation. On successful cancellation, + // the operation is not deleted; instead, it becomes an operation with + // an [Operation.error][google.longrunning.Operation.error] value with a + // [google.rpc.Status.code][google.rpc.Status.code] of `1`, corresponding to + // `Code.CANCELLED`. + rpc CancelOperation(CancelOperationRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + post: "/v1/{name=operations/**}:cancel" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + + // Waits until the specified long-running operation is done or reaches at most + // a specified timeout, returning the latest state. If the operation is + // already done, the latest state is immediately returned. If the timeout + // specified is greater than the default HTTP/RPC timeout, the HTTP/RPC + // timeout is used. If the server does not support this method, it returns + // `google.rpc.Code.UNIMPLEMENTED`. + // Note that this method is on a best-effort basis. It may return the latest + // state before the specified timeout (including immediately), meaning even an + // immediate response is no guarantee that the operation is done. + rpc WaitOperation(WaitOperationRequest) returns (Operation) {} +} + +// This resource represents a long-running operation that is the result of a +// network API call. +message Operation { + // The server-assigned name, which is only unique within the same service that + // originally returns it. If you use the default HTTP mapping, the + // `name` should be a resource name ending with `operations/{unique_id}`. + string name = 1; + + // Service-specific metadata associated with the operation. It typically + // contains progress information and common metadata such as create time. + // Some services might not provide such metadata. Any method that returns a + // long-running operation should document the metadata type, if any. + google.protobuf.Any metadata = 2; + + // If the value is `false`, it means the operation is still in progress. + // If `true`, the operation is completed, and either `error` or `response` is + // available. + bool done = 3; + + // The operation result, which can be either an `error` or a valid `response`. + // If `done` == `false`, neither `error` nor `response` is set. + // If `done` == `true`, exactly one of `error` or `response` can be set. + // Some services might not provide the result. + oneof result { + // The error result of the operation in case of failure or cancellation. + google.rpc.Status error = 4; + + // The normal, successful response of the operation. If the original + // method returns no data on success, such as `Delete`, the response is + // `google.protobuf.Empty`. If the original method is standard + // `Get`/`Create`/`Update`, the response should be the resource. For other + // methods, the response should have the type `XxxResponse`, where `Xxx` + // is the original method name. For example, if the original method name + // is `TakeSnapshot()`, the inferred response type is + // `TakeSnapshotResponse`. + google.protobuf.Any response = 5; + } +} + +// The request message for +// [Operations.GetOperation][google.longrunning.Operations.GetOperation]. +message GetOperationRequest { + // The name of the operation resource. + string name = 1; +} + +// The request message for +// [Operations.ListOperations][google.longrunning.Operations.ListOperations]. +message ListOperationsRequest { + // The name of the operation's parent resource. + string name = 4; + + // The standard list filter. + string filter = 1; + + // The standard list page size. + int32 page_size = 2; + + // The standard list page token. + string page_token = 3; +} + +// The response message for +// [Operations.ListOperations][google.longrunning.Operations.ListOperations]. +message ListOperationsResponse { + // A list of operations that matches the specified filter in the request. + repeated Operation operations = 1; + + // The standard List next-page token. + string next_page_token = 2; +} + +// The request message for +// [Operations.CancelOperation][google.longrunning.Operations.CancelOperation]. +message CancelOperationRequest { + // The name of the operation resource to be cancelled. + string name = 1; +} + +// The request message for +// [Operations.DeleteOperation][google.longrunning.Operations.DeleteOperation]. +message DeleteOperationRequest { + // The name of the operation resource to be deleted. + string name = 1; +} + +// The request message for +// [Operations.WaitOperation][google.longrunning.Operations.WaitOperation]. +message WaitOperationRequest { + // The name of the operation resource to wait on. + string name = 1; + + // The maximum duration to wait before timing out. If left blank, the wait + // will be at most the time permitted by the underlying HTTP/RPC protocol. + // If RPC context deadline is also specified, the shorter one will be used. + google.protobuf.Duration timeout = 2; +} + +// A message representing the message types used by a long-running operation. +// +// Example: +// +// rpc Export(ExportRequest) returns (google.longrunning.Operation) { +// option (google.longrunning.operation_info) = { +// response_type: "ExportResponse" +// metadata_type: "ExportMetadata" +// }; +// } +message OperationInfo { + // Required. The message name of the primary return type for this + // long-running operation. + // This type will be used to deserialize the LRO's response. + // + // If the response is in a different package from the rpc, a fully-qualified + // message name must be used (e.g. `google.protobuf.Struct`). + // + // Note: Altering this value constitutes a breaking change. + string response_type = 1; + + // Required. The message name of the metadata type for this long-running + // operation. + // + // If the response is in a different package from the rpc, a fully-qualified + // message name must be used (e.g. `google.protobuf.Struct`). + // + // Note: Altering this value constitutes a breaking change. + string metadata_type = 2; +} diff --git a/proto/google/protobuf/any.proto b/proto/google/protobuf/any.proto new file mode 100644 index 0000000..e2c2042 --- /dev/null +++ b/proto/google/protobuf/any.proto @@ -0,0 +1,158 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/anypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// Example 4: Pack and unpack a message in Go +// +// foo := &pb.Foo{...} +// any, err := anypb.New(foo) +// if err != nil { +// ... +// } +// ... +// foo := &pb.Foo{} +// if err := any.UnmarshalTo(foo); err != nil { +// ... +// } +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// +// JSON +// +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name that uniquely identifies the type of the serialized + // protocol buffer message. This string must contain at least + // one "/" character. The last segment of the URL's path must represent + // the fully qualified name of the type (as in + // `path/google.protobuf.Duration`). The name should be in a canonical form + // (e.g., leading "." is not accepted). + // + // In practice, teams usually precompile into the binary all types that they + // expect it to use in the context of Any. However, for URLs which use the + // scheme `http`, `https`, or no scheme, one can optionally set up a type + // server that maps type URLs to message definitions as follows: + // + // * If no scheme is provided, `https` is assumed. + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Note: this functionality is not currently available in the official + // protobuf release, and it is not used for type URLs beginning with + // type.googleapis.com. + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + // + string type_url = 1; + + // Must be a valid serialized protocol buffer of the above specified type. + bytes value = 2; +} diff --git a/proto/google/protobuf/api.proto b/proto/google/protobuf/api.proto new file mode 100644 index 0000000..3d598fc --- /dev/null +++ b/proto/google/protobuf/api.proto @@ -0,0 +1,208 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/source_context.proto"; +import "google/protobuf/type.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "ApiProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/apipb"; + +// Api is a light-weight descriptor for an API Interface. +// +// Interfaces are also described as "protocol buffer services" in some contexts, +// such as by the "service" keyword in a .proto file, but they are different +// from API Services, which represent a concrete implementation of an interface +// as opposed to simply a description of methods and bindings. They are also +// sometimes simply referred to as "APIs" in other contexts, such as the name of +// this message itself. See https://cloud.google.com/apis/design/glossary for +// detailed terminology. +message Api { + // The fully qualified name of this interface, including package name + // followed by the interface's simple name. + string name = 1; + + // The methods of this interface, in unspecified order. + repeated Method methods = 2; + + // Any metadata attached to the interface. + repeated Option options = 3; + + // A version string for this interface. If specified, must have the form + // `major-version.minor-version`, as in `1.10`. If the minor version is + // omitted, it defaults to zero. If the entire version field is empty, the + // major version is derived from the package name, as outlined below. If the + // field is not empty, the version in the package name will be verified to be + // consistent with what is provided here. + // + // The versioning schema uses [semantic + // versioning](http://semver.org) where the major version number + // indicates a breaking change and the minor version an additive, + // non-breaking change. Both version numbers are signals to users + // what to expect from different versions, and should be carefully + // chosen based on the product plan. + // + // The major version is also reflected in the package name of the + // interface, which must end in `v`, as in + // `google.feature.v1`. For major versions 0 and 1, the suffix can + // be omitted. Zero major versions must only be used for + // experimental, non-GA interfaces. + // + // + string version = 4; + + // Source context for the protocol buffer service represented by this + // message. + SourceContext source_context = 5; + + // Included interfaces. See [Mixin][]. + repeated Mixin mixins = 6; + + // The source syntax of the service. + Syntax syntax = 7; +} + +// Method represents a method of an API interface. +message Method { + // The simple name of this method. + string name = 1; + + // A URL of the input message type. + string request_type_url = 2; + + // If true, the request is streamed. + bool request_streaming = 3; + + // The URL of the output message type. + string response_type_url = 4; + + // If true, the response is streamed. + bool response_streaming = 5; + + // Any metadata attached to the method. + repeated Option options = 6; + + // The source syntax of this method. + Syntax syntax = 7; +} + +// Declares an API Interface to be included in this interface. The including +// interface must redeclare all the methods from the included interface, but +// documentation and options are inherited as follows: +// +// - If after comment and whitespace stripping, the documentation +// string of the redeclared method is empty, it will be inherited +// from the original method. +// +// - Each annotation belonging to the service config (http, +// visibility) which is not set in the redeclared method will be +// inherited. +// +// - If an http annotation is inherited, the path pattern will be +// modified as follows. Any version prefix will be replaced by the +// version of the including interface plus the [root][] path if +// specified. +// +// Example of a simple mixin: +// +// package google.acl.v1; +// service AccessControl { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v1/{resource=**}:getAcl"; +// } +// } +// +// package google.storage.v2; +// service Storage { +// rpc GetAcl(GetAclRequest) returns (Acl); +// +// // Get a data record. +// rpc GetData(GetDataRequest) returns (Data) { +// option (google.api.http).get = "/v2/{resource=**}"; +// } +// } +// +// Example of a mixin configuration: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// +// The mixin construct implies that all methods in `AccessControl` are +// also declared with same name and request/response types in +// `Storage`. A documentation generator or annotation processor will +// see the effective `Storage.GetAcl` method after inheriting +// documentation and annotations as follows: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/{resource=**}:getAcl"; +// } +// ... +// } +// +// Note how the version in the path pattern changed from `v1` to `v2`. +// +// If the `root` field in the mixin is specified, it should be a +// relative path under which inherited HTTP paths are placed. Example: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// root: acls +// +// This implies the following inherited HTTP annotation: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/acls/{resource=**}:getAcl"; +// } +// ... +// } +message Mixin { + // The fully qualified name of the interface which is included. + string name = 1; + + // If non-empty specifies a path under which inherited HTTP paths + // are rooted. + string root = 2; +} diff --git a/proto/google/protobuf/compiler/plugin.proto b/proto/google/protobuf/compiler/plugin.proto new file mode 100644 index 0000000..9242aac --- /dev/null +++ b/proto/google/protobuf/compiler/plugin.proto @@ -0,0 +1,183 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// +// WARNING: The plugin interface is currently EXPERIMENTAL and is subject to +// change. +// +// protoc (aka the Protocol Compiler) can be extended via plugins. A plugin is +// just a program that reads a CodeGeneratorRequest from stdin and writes a +// CodeGeneratorResponse to stdout. +// +// Plugins written using C++ can use google/protobuf/compiler/plugin.h instead +// of dealing with the raw protocol defined here. +// +// A plugin executable needs only to be placed somewhere in the path. The +// plugin should be named "protoc-gen-$NAME", and will then be used when the +// flag "--${NAME}_out" is passed to protoc. + +syntax = "proto2"; + +package google.protobuf.compiler; +option java_package = "com.google.protobuf.compiler"; +option java_outer_classname = "PluginProtos"; + +option go_package = "google.golang.org/protobuf/types/pluginpb"; + +import "google/protobuf/descriptor.proto"; + +// The version number of protocol compiler. +message Version { + optional int32 major = 1; + optional int32 minor = 2; + optional int32 patch = 3; + // A suffix for alpha, beta or rc release, e.g., "alpha-1", "rc2". It should + // be empty for mainline stable releases. + optional string suffix = 4; +} + +// An encoded CodeGeneratorRequest is written to the plugin's stdin. +message CodeGeneratorRequest { + // The .proto files that were explicitly listed on the command-line. The + // code generator should generate code only for these files. Each file's + // descriptor will be included in proto_file, below. + repeated string file_to_generate = 1; + + // The generator parameter passed on the command-line. + optional string parameter = 2; + + // FileDescriptorProtos for all files in files_to_generate and everything + // they import. The files will appear in topological order, so each file + // appears before any file that imports it. + // + // protoc guarantees that all proto_files will be written after + // the fields above, even though this is not technically guaranteed by the + // protobuf wire format. This theoretically could allow a plugin to stream + // in the FileDescriptorProtos and handle them one by one rather than read + // the entire set into memory at once. However, as of this writing, this + // is not similarly optimized on protoc's end -- it will store all fields in + // memory at once before sending them to the plugin. + // + // Type names of fields and extensions in the FileDescriptorProto are always + // fully qualified. + repeated FileDescriptorProto proto_file = 15; + + // The version number of protocol compiler. + optional Version compiler_version = 3; + +} + +// The plugin writes an encoded CodeGeneratorResponse to stdout. +message CodeGeneratorResponse { + // Error message. If non-empty, code generation failed. The plugin process + // should exit with status code zero even if it reports an error in this way. + // + // This should be used to indicate errors in .proto files which prevent the + // code generator from generating correct code. Errors which indicate a + // problem in protoc itself -- such as the input CodeGeneratorRequest being + // unparseable -- should be reported by writing a message to stderr and + // exiting with a non-zero status code. + optional string error = 1; + + // A bitmask of supported features that the code generator supports. + // This is a bitwise "or" of values from the Feature enum. + optional uint64 supported_features = 2; + + // Sync with code_generator.h. + enum Feature { + FEATURE_NONE = 0; + FEATURE_PROTO3_OPTIONAL = 1; + } + + // Represents a single generated file. + message File { + // The file name, relative to the output directory. The name must not + // contain "." or ".." components and must be relative, not be absolute (so, + // the file cannot lie outside the output directory). "/" must be used as + // the path separator, not "\". + // + // If the name is omitted, the content will be appended to the previous + // file. This allows the generator to break large files into small chunks, + // and allows the generated text to be streamed back to protoc so that large + // files need not reside completely in memory at one time. Note that as of + // this writing protoc does not optimize for this -- it will read the entire + // CodeGeneratorResponse before writing files to disk. + optional string name = 1; + + // If non-empty, indicates that the named file should already exist, and the + // content here is to be inserted into that file at a defined insertion + // point. This feature allows a code generator to extend the output + // produced by another code generator. The original generator may provide + // insertion points by placing special annotations in the file that look + // like: + // @@protoc_insertion_point(NAME) + // The annotation can have arbitrary text before and after it on the line, + // which allows it to be placed in a comment. NAME should be replaced with + // an identifier naming the point -- this is what other generators will use + // as the insertion_point. Code inserted at this point will be placed + // immediately above the line containing the insertion point (thus multiple + // insertions to the same point will come out in the order they were added). + // The double-@ is intended to make it unlikely that the generated code + // could contain things that look like insertion points by accident. + // + // For example, the C++ code generator places the following line in the + // .pb.h files that it generates: + // // @@protoc_insertion_point(namespace_scope) + // This line appears within the scope of the file's package namespace, but + // outside of any particular class. Another plugin can then specify the + // insertion_point "namespace_scope" to generate additional classes or + // other declarations that should be placed in this scope. + // + // Note that if the line containing the insertion point begins with + // whitespace, the same whitespace will be added to every line of the + // inserted text. This is useful for languages like Python, where + // indentation matters. In these languages, the insertion point comment + // should be indented the same amount as any inserted code will need to be + // in order to work correctly in that context. + // + // The code generator that generates the initial file and the one which + // inserts into it must both run as part of a single invocation of protoc. + // Code generators are executed in the order in which they appear on the + // command line. + // + // If |insertion_point| is present, |name| must also be present. + optional string insertion_point = 2; + + // The file contents. + optional string content = 15; + + // Information describing the file content being inserted. If an insertion + // point is used, this information will be appropriately offset and inserted + // into the code generation metadata for the generated files. + optional GeneratedCodeInfo generated_code_info = 16; + } + repeated File file = 15; +} diff --git a/proto/google/protobuf/descriptor.proto b/proto/google/protobuf/descriptor.proto new file mode 100644 index 0000000..f8eb216 --- /dev/null +++ b/proto/google/protobuf/descriptor.proto @@ -0,0 +1,921 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// The messages in this file describe the definitions found in .proto files. +// A valid .proto file can be translated directly to a FileDescriptorProto +// without any other information (e.g. without reading its imports). + + +syntax = "proto2"; + +package google.protobuf; + +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DescriptorProtos"; +option csharp_namespace = "Google.Protobuf.Reflection"; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// descriptor.proto must be optimized for speed because reflection-based +// algorithms don't work during bootstrapping. +option optimize_for = SPEED; + +// The protocol compiler can output a FileDescriptorSet containing the .proto +// files it parses. +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; +} + +// Describes a complete .proto file. +message FileDescriptorProto { + optional string name = 1; // file name, relative to root of source tree + optional string package = 2; // e.g. "foo", "foo.bar", etc. + + // Names of files imported by this file. + repeated string dependency = 3; + // Indexes of the public imported files in the dependency list above. + repeated int32 public_dependency = 10; + // Indexes of the weak imported files in the dependency list. + // For Google-internal migration only. Do not use. + repeated int32 weak_dependency = 11; + + // All top-level definitions in this file. + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + + optional FileOptions options = 8; + + // This field contains optional information about the original source code. + // You may safely remove this entire field without harming runtime + // functionality of the descriptors -- the information is needed only by + // development tools. + optional SourceCodeInfo source_code_info = 9; + + // The syntax of the proto file. + // The supported values are "proto2" and "proto3". + optional string syntax = 12; +} + +// Describes a message type. +message DescriptorProto { + optional string name = 1; + + repeated FieldDescriptorProto field = 2; + repeated FieldDescriptorProto extension = 6; + + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + + message ExtensionRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + + optional ExtensionRangeOptions options = 3; + } + repeated ExtensionRange extension_range = 5; + + repeated OneofDescriptorProto oneof_decl = 8; + + optional MessageOptions options = 7; + + // Range of reserved tag numbers. Reserved tag numbers may not be used by + // fields or extension ranges in the same message. Reserved ranges may + // not overlap. + message ReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + } + repeated ReservedRange reserved_range = 9; + // Reserved field names, which may not be used by fields in the same message. + // A given name may only be reserved once. + repeated string reserved_name = 10; +} + +message ExtensionRangeOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +// Describes a field within a message. +message FieldDescriptorProto { + enum Type { + // 0 is reserved for errors. + // Order is weird for historical reasons. + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + // negative values are likely. + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + // negative values are likely. + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + // Tag-delimited aggregate. + // Group type is deprecated and not supported in proto3. However, Proto3 + // implementations should still be able to parse the group wire format and + // treat group fields as unknown fields. + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; // Length-delimited aggregate. + + // New in version 2. + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; // Uses ZigZag encoding. + TYPE_SINT64 = 18; // Uses ZigZag encoding. + } + + enum Label { + // 0 is reserved for errors + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + + optional string name = 1; + optional int32 number = 3; + optional Label label = 4; + + // If type_name is set, this need not be set. If both this and type_name + // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. + optional Type type = 5; + + // For message and enum types, this is the name of the type. If the name + // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + // rules are used to find the type (i.e. first the nested types within this + // message are searched, then within the parent, on up to the root + // namespace). + optional string type_name = 6; + + // For extensions, this is the name of the type being extended. It is + // resolved in the same manner as type_name. + optional string extendee = 2; + + // For numeric types, contains the original text representation of the value. + // For booleans, "true" or "false". + // For strings, contains the default text contents (not escaped in any way). + // For bytes, contains the C escaped value. All bytes >= 128 are escaped. + optional string default_value = 7; + + // If set, gives the index of a oneof in the containing type's oneof_decl + // list. This field is a member of that oneof. + optional int32 oneof_index = 9; + + // JSON name of this field. The value is set by protocol compiler. If the + // user has set a "json_name" option on this field, that option's value + // will be used. Otherwise, it's deduced from the field's name by converting + // it to camelCase. + optional string json_name = 10; + + optional FieldOptions options = 8; + + // If true, this is a proto3 "optional". When a proto3 field is optional, it + // tracks presence regardless of field type. + // + // When proto3_optional is true, this field must be belong to a oneof to + // signal to old proto3 clients that presence is tracked for this field. This + // oneof is known as a "synthetic" oneof, and this field must be its sole + // member (each proto3 optional field gets its own synthetic oneof). Synthetic + // oneofs exist in the descriptor only, and do not generate any API. Synthetic + // oneofs must be ordered after all "real" oneofs. + // + // For message fields, proto3_optional doesn't create any semantic change, + // since non-repeated message fields always track presence. However it still + // indicates the semantic detail of whether the user wrote "optional" or not. + // This can be useful for round-tripping the .proto file. For consistency we + // give message fields a synthetic oneof also, even though it is not required + // to track presence. This is especially important because the parser can't + // tell if a field is a message or an enum, so it must always create a + // synthetic oneof. + // + // Proto2 optional fields do not set this flag, because they already indicate + // optional with `LABEL_OPTIONAL`. + optional bool proto3_optional = 17; +} + +// Describes a oneof. +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} + +// Describes an enum type. +message EnumDescriptorProto { + optional string name = 1; + + repeated EnumValueDescriptorProto value = 2; + + optional EnumOptions options = 3; + + // Range of reserved numeric values. Reserved values may not be used by + // entries in the same enum. Reserved ranges may not overlap. + // + // Note that this is distinct from DescriptorProto.ReservedRange in that it + // is inclusive such that it can appropriately represent the entire int32 + // domain. + message EnumReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Inclusive. + } + + // Range of reserved numeric values. Reserved numeric values may not be used + // by enum values in the same enum declaration. Reserved ranges may not + // overlap. + repeated EnumReservedRange reserved_range = 4; + + // Reserved enum value names, which may not be reused. A given name may only + // be reserved once. + repeated string reserved_name = 5; +} + +// Describes a value within an enum. +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + + optional EnumValueOptions options = 3; +} + +// Describes a service. +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + + optional ServiceOptions options = 3; +} + +// Describes a method of a service. +message MethodDescriptorProto { + optional string name = 1; + + // Input and output type names. These are resolved in the same way as + // FieldDescriptorProto.type_name, but must refer to a message type. + optional string input_type = 2; + optional string output_type = 3; + + optional MethodOptions options = 4; + + // Identifies if client streams multiple client messages + optional bool client_streaming = 5 [default = false]; + // Identifies if server streams multiple server messages + optional bool server_streaming = 6 [default = false]; +} + + +// =================================================================== +// Options + +// Each of the definitions above may have "options" attached. These are +// just annotations which may cause code to be generated slightly differently +// or may contain hints for code that manipulates protocol messages. +// +// Clients may define custom options as extensions of the *Options messages. +// These extensions may not yet be known at parsing time, so the parser cannot +// store the values in them. Instead it stores them in a field in the *Options +// message called uninterpreted_option. This field must have the same name +// across all *Options messages. We then use this field to populate the +// extensions when we build a descriptor, at which point all protos have been +// parsed and so all extensions are known. +// +// Extension numbers for custom options may be chosen as follows: +// * For options which will only be used within a single application or +// organization, or for experimental options, use field numbers 50000 +// through 99999. It is up to you to ensure that you do not use the +// same number for multiple options. +// * For options which will be published and used publicly by multiple +// independent entities, e-mail protobuf-global-extension-registry@google.com +// to reserve extension numbers. Simply provide your project name (e.g. +// Objective-C plugin) and your project website (if available) -- there's no +// need to explain how you intend to use them. Usually you only need one +// extension number. You can declare multiple options with only one extension +// number by putting them in a sub-message. See the Custom Options section of +// the docs for examples: +// https://developers.google.com/protocol-buffers/docs/proto#options +// If this turns out to be popular, a web service will be set up +// to automatically assign option numbers. + +message FileOptions { + + // Sets the Java package where classes generated from this .proto will be + // placed. By default, the proto package is used, but this is often + // inappropriate because proto packages do not normally start with backwards + // domain names. + optional string java_package = 1; + + + // Controls the name of the wrapper Java class generated for the .proto file. + // That class will always contain the .proto file's getDescriptor() method as + // well as any top-level extensions defined in the .proto file. + // If java_multiple_files is disabled, then all the other classes from the + // .proto file will be nested inside the single wrapper outer class. + optional string java_outer_classname = 8; + + // If enabled, then the Java code generator will generate a separate .java + // file for each top-level message, enum, and service defined in the .proto + // file. Thus, these types will *not* be nested inside the wrapper class + // named by java_outer_classname. However, the wrapper class will still be + // generated to contain the file's getDescriptor() method as well as any + // top-level extensions defined in the file. + optional bool java_multiple_files = 10 [default = false]; + + // This option does nothing. + optional bool java_generate_equals_and_hash = 20 [deprecated=true]; + + // If set true, then the Java2 code generator will generate code that + // throws an exception whenever an attempt is made to assign a non-UTF-8 + // byte sequence to a string field. + // Message reflection will do the same. + // However, an extension field still accepts non-UTF-8 byte sequences. + // This option has no effect on when used with the lite runtime. + optional bool java_string_check_utf8 = 27 [default = false]; + + + // Generated classes can be optimized for speed or code size. + enum OptimizeMode { + SPEED = 1; // Generate complete code for parsing, serialization, + // etc. + CODE_SIZE = 2; // Use ReflectionOps to implement these methods. + LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. + } + optional OptimizeMode optimize_for = 9 [default = SPEED]; + + // Sets the Go package where structs generated from this .proto will be + // placed. If omitted, the Go package will be derived from the following: + // - The basename of the package import path, if provided. + // - Otherwise, the package statement in the .proto file, if present. + // - Otherwise, the basename of the .proto file, without extension. + optional string go_package = 11; + + + + + // Should generic services be generated in each language? "Generic" services + // are not specific to any particular RPC system. They are generated by the + // main code generators in each language (without additional plugins). + // Generic services were the only kind of service generation supported by + // early versions of google.protobuf. + // + // Generic services are now considered deprecated in favor of using plugins + // that generate code specific to your particular RPC system. Therefore, + // these default to false. Old code which depends on generic services should + // explicitly set them to true. + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool php_generic_services = 42 [default = false]; + + // Is this file deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for everything in the file, or it will be completely ignored; in the very + // least, this is a formalization for deprecating files. + optional bool deprecated = 23 [default = false]; + + // Enables the use of arenas for the proto messages in this file. This applies + // only to generated classes for C++. + optional bool cc_enable_arenas = 31 [default = true]; + + + // Sets the objective c class prefix which is prepended to all objective c + // generated classes from this .proto. There is no default. + optional string objc_class_prefix = 36; + + // Namespace for generated classes; defaults to the package. + optional string csharp_namespace = 37; + + // By default Swift generators will take the proto package and CamelCase it + // replacing '.' with underscore and use that to prefix the types/symbols + // defined. When this options is provided, they will use this value instead + // to prefix the types/symbols defined. + optional string swift_prefix = 39; + + // Sets the php class prefix which is prepended to all php generated classes + // from this .proto. Default is empty. + optional string php_class_prefix = 40; + + // Use this option to change the namespace of php generated classes. Default + // is empty. When this option is empty, the package name will be used for + // determining the namespace. + optional string php_namespace = 41; + + // Use this option to change the namespace of php generated metadata classes. + // Default is empty. When this option is empty, the proto file name will be + // used for determining the namespace. + optional string php_metadata_namespace = 44; + + // Use this option to change the package of ruby generated classes. Default + // is empty. When this option is not set, the package name will be used for + // determining the ruby package. + optional string ruby_package = 45; + + + // The parser stores options it doesn't recognize here. + // See the documentation for the "Options" section above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. + // See the documentation for the "Options" section above. + extensions 1000 to max; + + reserved 38; +} + +message MessageOptions { + // Set true to use the old proto1 MessageSet wire format for extensions. + // This is provided for backwards-compatibility with the MessageSet wire + // format. You should not use this for any other reason: It's less + // efficient, has fewer features, and is more complicated. + // + // The message must be defined exactly as follows: + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // Note that the message cannot have any defined fields; MessageSets only + // have extensions. + // + // All extensions of your type must be singular messages; e.g. they cannot + // be int32s, enums, or repeated messages. + // + // Because this is an option, the above two restrictions are not enforced by + // the protocol compiler. + optional bool message_set_wire_format = 1 [default = false]; + + // Disables the generation of the standard "descriptor()" accessor, which can + // conflict with a field of the same name. This is meant to make migration + // from proto1 easier; new code should avoid fields named "descriptor". + optional bool no_standard_descriptor_accessor = 2 [default = false]; + + // Is this message deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the message, or it will be completely ignored; in the very least, + // this is a formalization for deprecating messages. + optional bool deprecated = 3 [default = false]; + + reserved 4, 5, 6; + + // Whether the message is an automatically generated map entry type for the + // maps field. + // + // For maps fields: + // map map_field = 1; + // The parsed descriptor looks like: + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; + // + // Implementations may choose not to generate the map_entry=true message, but + // use a native map in the target language to hold the keys and values. + // The reflection APIs in such implementations still need to work as + // if the field is a repeated message field. + // + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + optional bool map_entry = 7; + + reserved 8; // javalite_serializable + reserved 9; // javanano_as_lite + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message FieldOptions { + // The ctype option instructs the C++ code generator to use a different + // representation of the field than it normally would. See the specific + // options below. This option is not yet implemented in the open source + // release -- sorry, we'll try to include it in a future version! + optional CType ctype = 1 [default = STRING]; + enum CType { + // Default mode. + STRING = 0; + + CORD = 1; + + STRING_PIECE = 2; + } + // The packed option can be enabled for repeated primitive fields to enable + // a more efficient representation on the wire. Rather than repeatedly + // writing the tag and type for each element, the entire array is encoded as + // a single length-delimited blob. In proto3, only explicit setting it to + // false will avoid using packed encoding. + optional bool packed = 2; + + // The jstype option determines the JavaScript type used for values of the + // field. The option is permitted only for 64 bit integral and fixed types + // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + // is represented as JavaScript string, which avoids loss of precision that + // can happen when a large value is converted to a floating point JavaScript. + // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + // use the JavaScript "number" type. The behavior of the default option + // JS_NORMAL is implementation dependent. + // + // This option is an enum to permit additional types to be added, e.g. + // goog.math.Integer. + optional JSType jstype = 6 [default = JS_NORMAL]; + enum JSType { + // Use the default type. + JS_NORMAL = 0; + + // Use JavaScript strings. + JS_STRING = 1; + + // Use JavaScript numbers. + JS_NUMBER = 2; + } + + // Should this field be parsed lazily? Lazy applies only to message-type + // fields. It means that when the outer message is initially parsed, the + // inner message's contents will not be parsed but instead stored in encoded + // form. The inner message will actually be parsed when it is first accessed. + // + // This is only a hint. Implementations are free to choose whether to use + // eager or lazy parsing regardless of the value of this option. However, + // setting this option true suggests that the protocol author believes that + // using lazy parsing on this field is worth the additional bookkeeping + // overhead typically needed to implement it. + // + // This option does not affect the public interface of any generated code; + // all method signatures remain the same. Furthermore, thread-safety of the + // interface is not affected by this option; const methods remain safe to + // call from multiple threads concurrently, while non-const methods continue + // to require exclusive access. + // + // + // Note that implementations may choose not to check required fields within + // a lazy sub-message. That is, calling IsInitialized() on the outer message + // may return true even if the inner message has missing required fields. + // This is necessary because otherwise the inner message would have to be + // parsed in order to perform the check, defeating the purpose of lazy + // parsing. An implementation which chooses not to check required fields + // must be consistent about it. That is, for any particular sub-message, the + // implementation must either *always* check its required fields, or *never* + // check its required fields, regardless of whether or not the message has + // been parsed. + // + // As of 2021, lazy does no correctness checks on the byte stream during + // parsing. This may lead to crashes if and when an invalid byte stream is + // finally parsed upon access. + // + // TODO(b/211906113): Enable validation on lazy fields. + optional bool lazy = 5 [default = false]; + + // unverified_lazy does no correctness checks on the byte stream. This should + // only be used where lazy with verification is prohibitive for performance + // reasons. + optional bool unverified_lazy = 15 [default = false]; + + // Is this field deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for accessors, or it will be completely ignored; in the very least, this + // is a formalization for deprecating fields. + optional bool deprecated = 3 [default = false]; + + // For Google-internal migration only. Do not use. + optional bool weak = 10 [default = false]; + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; + + reserved 4; // removed jtype +} + +message OneofOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumOptions { + + // Set this option to true to allow mapping different tag names to the same + // value. + optional bool allow_alias = 2; + + // Is this enum deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum, or it will be completely ignored; in the very least, this + // is a formalization for deprecating enums. + optional bool deprecated = 3 [default = false]; + + reserved 5; // javanano_as_lite + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumValueOptions { + // Is this enum value deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum value, or it will be completely ignored; in the very least, + // this is a formalization for deprecating enum values. + optional bool deprecated = 1 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message ServiceOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this service deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the service, or it will be completely ignored; in the very least, + // this is a formalization for deprecating services. + optional bool deprecated = 33 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MethodOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this method deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the method, or it will be completely ignored; in the very least, + // this is a formalization for deprecating methods. + optional bool deprecated = 33 [default = false]; + + // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, + // or neither? HTTP based RPC implementation may choose GET verb for safe + // methods, and PUT verb for idempotent methods instead of the default POST. + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; // implies idempotent + IDEMPOTENT = 2; // idempotent, but may have side effects + } + optional IdempotencyLevel idempotency_level = 34 + [default = IDEMPOTENCY_UNKNOWN]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + + +// A message representing a option the parser does not recognize. This only +// appears in options protos created by the compiler::Parser class. +// DescriptorPool resolves these when building Descriptor objects. Therefore, +// options protos in descriptor objects (e.g. returned by Descriptor::options(), +// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions +// in them. +message UninterpretedOption { + // The name of the uninterpreted option. Each string represents a segment in + // a dot-separated name. is_extension is true iff a segment represents an + // extension (denoted with parentheses in options specs in .proto files). + // E.g.,{ ["foo", false], ["bar.baz", true], ["moo", false] } represents + // "foo.(bar.baz).moo". + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } + repeated NamePart name = 2; + + // The value of the uninterpreted option, in whatever type the tokenizer + // identified it as during parsing. Exactly one of these should be set. + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; +} + +// =================================================================== +// Optional source code info + +// Encapsulates information about the original source file from which a +// FileDescriptorProto was generated. +message SourceCodeInfo { + // A Location identifies a piece of source code in a .proto file which + // corresponds to a particular definition. This information is intended + // to be useful to IDEs, code indexers, documentation generators, and similar + // tools. + // + // For example, say we have a file like: + // message Foo { + // optional string foo = 1; + // } + // Let's look at just the field definition: + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // We have the following locations: + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // Notes: + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendant. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. + repeated Location location = 1; + message Location { + // Identifies which part of the FileDescriptorProto was defined at this + // location. + // + // Each element is a field number or an index. They form a path from + // the root FileDescriptorProto to the place where the definition occurs. + // For example, this path: + // [ 4, 3, 2, 7, 1 ] + // refers to: + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // This is because FileDescriptorProto.message_type has field number 4: + // repeated DescriptorProto message_type = 4; + // and DescriptorProto.field has field number 2: + // repeated FieldDescriptorProto field = 2; + // and FieldDescriptorProto.name has field number 1: + // optional string name = 1; + // + // Thus, the above path gives the location of a field name. If we removed + // the last element: + // [ 4, 3, 2, 7 ] + // this path refers to the whole field declaration (from the beginning + // of the label to the terminating semicolon). + repeated int32 path = 1 [packed = true]; + + // Always has exactly three or four elements: start line, start column, + // end line (optional, otherwise assumed same as start line), end column. + // These are packed into a single field for efficiency. Note that line + // and column numbers are zero-based -- typically you will want to add + // 1 to each before displaying to a user. + repeated int32 span = 2 [packed = true]; + + // If this SourceCodeInfo represents a complete declaration, these are any + // comments appearing before and after the declaration which appear to be + // attached to the declaration. + // + // A series of line comments appearing on consecutive lines, with no other + // tokens appearing on those lines, will be treated as a single comment. + // + // leading_detached_comments will keep paragraphs of comments that appear + // before (but not connected to) the current element. Each paragraph, + // separated by empty lines, will be one comment element in the repeated + // field. + // + // Only the comment content is provided; comment markers (e.g. //) are + // stripped out. For block comments, leading whitespace and an asterisk + // will be stripped from the beginning of each line other than the first. + // Newlines are included in the output. + // + // Examples: + // + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; + // + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. + // + // // Comment attached to moo. + // // + // // Another line attached to moo. + // optional double moo = 4; + // + // // Detached comment for corge. This is not leading or trailing comments + // // to moo or corge because there are blank lines separating it from + // // both. + // + // // Detached comment for corge paragraph 2. + // + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; + // + // // ignored detached comments. + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } +} + +// Describes the relationship between generated code and its original source +// file. A GeneratedCodeInfo message is associated with only one generated +// source file, but may contain references to different source .proto files. +message GeneratedCodeInfo { + // An Annotation connects some span of text in generated code to an element + // of its generating .proto file. + repeated Annotation annotation = 1; + message Annotation { + // Identifies the element in the original source .proto file. This field + // is formatted the same as SourceCodeInfo.Location.path. + repeated int32 path = 1 [packed = true]; + + // Identifies the filesystem path to the original source .proto. + optional string source_file = 2; + + // Identifies the starting offset in bytes in the generated code + // that relates to the identified object. + optional int32 begin = 3; + + // Identifies the ending offset in bytes in the generated code that + // relates to the identified offset. The end offset should be one past + // the last relevant byte (so the length of the text = end - begin). + optional int32 end = 4; + } +} diff --git a/proto/google/protobuf/duration.proto b/proto/google/protobuf/duration.proto new file mode 100644 index 0000000..81c3e36 --- /dev/null +++ b/proto/google/protobuf/duration.proto @@ -0,0 +1,116 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/durationpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DurationProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// # Examples +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (duration.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +// Example 3: Compute Duration from datetime.timedelta in Python. +// +// td = datetime.timedelta(days=3, minutes=10) +// duration = Duration() +// duration.FromTimedelta(td) +// +// # JSON Mapping +// +// In JSON format, the Duration type is encoded as a string rather than an +// object, where the string ends in the suffix "s" (indicating seconds) and +// is preceded by the number of seconds, with nanoseconds expressed as +// fractional seconds. For example, 3 seconds with 0 nanoseconds should be +// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should +// be expressed in JSON format as "3.000000001s", and 3 seconds and 1 +// microsecond should be expressed in JSON format as "3.000001s". +// +// +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. Note: these bounds are computed from: + // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} diff --git a/proto/google/protobuf/empty.proto b/proto/google/protobuf/empty.proto new file mode 100644 index 0000000..2227462 --- /dev/null +++ b/proto/google/protobuf/empty.proto @@ -0,0 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/emptypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "EmptyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// A generic empty message that you can re-use to avoid defining duplicated +// empty messages in your APIs. A typical example is to use it as the request +// or the response type of an API method. For instance: +// +// service Foo { +// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); +// } +// +message Empty {} diff --git a/proto/google/protobuf/field_mask.proto b/proto/google/protobuf/field_mask.proto new file mode 100644 index 0000000..6b5104f --- /dev/null +++ b/proto/google/protobuf/field_mask.proto @@ -0,0 +1,245 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "FieldMaskProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/fieldmaskpb"; +option cc_enable_arenas = true; + +// `FieldMask` represents a set of symbolic field paths, for example: +// +// paths: "f.a" +// paths: "f.b.d" +// +// Here `f` represents a field in some root message, `a` and `b` +// fields in the message found in `f`, and `d` a field found in the +// message in `f.b`. +// +// Field masks are used to specify a subset of fields that should be +// returned by a get operation or modified by an update operation. +// Field masks also have a custom JSON encoding (see below). +// +// # Field Masks in Projections +// +// When used in the context of a projection, a response message or +// sub-message is filtered by the API to only contain those fields as +// specified in the mask. For example, if the mask in the previous +// example is applied to a response message as follows: +// +// f { +// a : 22 +// b { +// d : 1 +// x : 2 +// } +// y : 13 +// } +// z: 8 +// +// The result will not contain specific values for fields x,y and z +// (their value will be set to the default, and omitted in proto text +// output): +// +// +// f { +// a : 22 +// b { +// d : 1 +// } +// } +// +// A repeated field is not allowed except at the last position of a +// paths string. +// +// If a FieldMask object is not present in a get operation, the +// operation applies to all fields (as if a FieldMask of all fields +// had been specified). +// +// Note that a field mask does not necessarily apply to the +// top-level response message. In case of a REST get operation, the +// field mask applies directly to the response, but in case of a REST +// list operation, the mask instead applies to each individual message +// in the returned resource list. In case of a REST custom method, +// other definitions may be used. Where the mask applies will be +// clearly documented together with its declaration in the API. In +// any case, the effect on the returned resource/resources is required +// behavior for APIs. +// +// # Field Masks in Update Operations +// +// A field mask in update operations specifies which fields of the +// targeted resource are going to be updated. The API is required +// to only change the values of the fields as specified in the mask +// and leave the others untouched. If a resource is passed in to +// describe the updated values, the API ignores the values of all +// fields not covered by the mask. +// +// If a repeated field is specified for an update operation, new values will +// be appended to the existing repeated field in the target resource. Note that +// a repeated field is only allowed in the last position of a `paths` string. +// +// If a sub-message is specified in the last position of the field mask for an +// update operation, then new value will be merged into the existing sub-message +// in the target resource. +// +// For example, given the target message: +// +// f { +// b { +// d: 1 +// x: 2 +// } +// c: [1] +// } +// +// And an update message: +// +// f { +// b { +// d: 10 +// } +// c: [2] +// } +// +// then if the field mask is: +// +// paths: ["f.b", "f.c"] +// +// then the result will be: +// +// f { +// b { +// d: 10 +// x: 2 +// } +// c: [1, 2] +// } +// +// An implementation may provide options to override this default behavior for +// repeated and message fields. +// +// In order to reset a field's value to the default, the field must +// be in the mask and set to the default value in the provided resource. +// Hence, in order to reset all fields of a resource, provide a default +// instance of the resource and set all fields in the mask, or do +// not provide a mask as described below. +// +// If a field mask is not present on update, the operation applies to +// all fields (as if a field mask of all fields has been specified). +// Note that in the presence of schema evolution, this may mean that +// fields the client does not know and has therefore not filled into +// the request will be reset to their default. If this is unwanted +// behavior, a specific service may require a client to always specify +// a field mask, producing an error if not. +// +// As with get operations, the location of the resource which +// describes the updated values in the request message depends on the +// operation kind. In any case, the effect of the field mask is +// required to be honored by the API. +// +// ## Considerations for HTTP REST +// +// The HTTP kind of an update operation which uses a field mask must +// be set to PATCH instead of PUT in order to satisfy HTTP semantics +// (PUT must only be used for full updates). +// +// # JSON Encoding of Field Masks +// +// In JSON, a field mask is encoded as a single string where paths are +// separated by a comma. Fields name in each path are converted +// to/from lower-camel naming conventions. +// +// As an example, consider the following message declarations: +// +// message Profile { +// User user = 1; +// Photo photo = 2; +// } +// message User { +// string display_name = 1; +// string address = 2; +// } +// +// In proto a field mask for `Profile` may look as such: +// +// mask { +// paths: "user.display_name" +// paths: "photo" +// } +// +// In JSON, the same mask is represented as below: +// +// { +// mask: "user.displayName,photo" +// } +// +// # Field Masks and Oneof Fields +// +// Field masks treat fields in oneofs just as regular fields. Consider the +// following message: +// +// message SampleMessage { +// oneof test_oneof { +// string name = 4; +// SubMessage sub_message = 9; +// } +// } +// +// The field mask can be: +// +// mask { +// paths: "name" +// } +// +// Or: +// +// mask { +// paths: "sub_message" +// } +// +// Note that oneof type names ("test_oneof" in this case) cannot be used in +// paths. +// +// ## Field Mask Verification +// +// The implementation of any API method which has a FieldMask type field in the +// request should verify the included field paths, and return an +// `INVALID_ARGUMENT` error if any path is unmappable. +message FieldMask { + // The set of field mask paths. + repeated string paths = 1; +} diff --git a/proto/google/protobuf/source_context.proto b/proto/google/protobuf/source_context.proto new file mode 100644 index 0000000..06bfc43 --- /dev/null +++ b/proto/google/protobuf/source_context.proto @@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "SourceContextProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/sourcecontextpb"; + +// `SourceContext` represents information about the source of a +// protobuf element, like the file in which it is defined. +message SourceContext { + // The path-qualified name of the .proto file that contained the associated + // protobuf element. For example: `"google/protobuf/source_context.proto"`. + string file_name = 1; +} diff --git a/proto/google/protobuf/struct.proto b/proto/google/protobuf/struct.proto new file mode 100644 index 0000000..0ac843c --- /dev/null +++ b/proto/google/protobuf/struct.proto @@ -0,0 +1,95 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/structpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "StructProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Struct` represents a structured data value, consisting of fields +// which map to dynamically typed values. In some languages, `Struct` +// might be supported by a native representation. For example, in +// scripting languages like JS a struct is represented as an +// object. The details of that representation are described together +// with the proto support for the language. +// +// The JSON representation for `Struct` is JSON object. +message Struct { + // Unordered map of dynamically typed values. + map fields = 1; +} + +// `Value` represents a dynamically typed value which can be either +// null, a number, a string, a boolean, a recursive struct value, or a +// list of values. A producer of value is expected to set one of these +// variants. Absence of any variant indicates an error. +// +// The JSON representation for `Value` is JSON value. +message Value { + // The kind of value. + oneof kind { + // Represents a null value. + NullValue null_value = 1; + // Represents a double value. + double number_value = 2; + // Represents a string value. + string string_value = 3; + // Represents a boolean value. + bool bool_value = 4; + // Represents a structured value. + Struct struct_value = 5; + // Represents a repeated `Value`. + ListValue list_value = 6; + } +} + +// `NullValue` is a singleton enumeration to represent the null value for the +// `Value` type union. +// +// The JSON representation for `NullValue` is JSON `null`. +enum NullValue { + // Null value. + NULL_VALUE = 0; +} + +// `ListValue` is a wrapper around a repeated field of values. +// +// The JSON representation for `ListValue` is JSON array. +message ListValue { + // Repeated field of dynamically typed values. + repeated Value values = 1; +} diff --git a/proto/google/protobuf/timestamp.proto b/proto/google/protobuf/timestamp.proto new file mode 100644 index 0000000..3b2df6d --- /dev/null +++ b/proto/google/protobuf/timestamp.proto @@ -0,0 +1,147 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/timestamppb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from Java `Instant.now()`. +// +// Instant now = Instant.now(); +// +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); +// +// +// Example 6: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/proto/google/protobuf/type.proto b/proto/google/protobuf/type.proto new file mode 100644 index 0000000..d3f6a68 --- /dev/null +++ b/proto/google/protobuf/type.proto @@ -0,0 +1,187 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/any.proto"; +import "google/protobuf/source_context.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TypeProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/typepb"; + +// A protocol buffer message type. +message Type { + // The fully qualified message name. + string name = 1; + // The list of fields. + repeated Field fields = 2; + // The list of types appearing in `oneof` definitions in this type. + repeated string oneofs = 3; + // The protocol buffer options. + repeated Option options = 4; + // The source context. + SourceContext source_context = 5; + // The source syntax. + Syntax syntax = 6; +} + +// A single field of a message type. +message Field { + // Basic field types. + enum Kind { + // Field type unknown. + TYPE_UNKNOWN = 0; + // Field type double. + TYPE_DOUBLE = 1; + // Field type float. + TYPE_FLOAT = 2; + // Field type int64. + TYPE_INT64 = 3; + // Field type uint64. + TYPE_UINT64 = 4; + // Field type int32. + TYPE_INT32 = 5; + // Field type fixed64. + TYPE_FIXED64 = 6; + // Field type fixed32. + TYPE_FIXED32 = 7; + // Field type bool. + TYPE_BOOL = 8; + // Field type string. + TYPE_STRING = 9; + // Field type group. Proto2 syntax only, and deprecated. + TYPE_GROUP = 10; + // Field type message. + TYPE_MESSAGE = 11; + // Field type bytes. + TYPE_BYTES = 12; + // Field type uint32. + TYPE_UINT32 = 13; + // Field type enum. + TYPE_ENUM = 14; + // Field type sfixed32. + TYPE_SFIXED32 = 15; + // Field type sfixed64. + TYPE_SFIXED64 = 16; + // Field type sint32. + TYPE_SINT32 = 17; + // Field type sint64. + TYPE_SINT64 = 18; + } + + // Whether a field is optional, required, or repeated. + enum Cardinality { + // For fields with unknown cardinality. + CARDINALITY_UNKNOWN = 0; + // For optional fields. + CARDINALITY_OPTIONAL = 1; + // For required fields. Proto2 syntax only. + CARDINALITY_REQUIRED = 2; + // For repeated fields. + CARDINALITY_REPEATED = 3; + } + + // The field type. + Kind kind = 1; + // The field cardinality. + Cardinality cardinality = 2; + // The field number. + int32 number = 3; + // The field name. + string name = 4; + // The field type URL, without the scheme, for message or enumeration + // types. Example: `"type.googleapis.com/google.protobuf.Timestamp"`. + string type_url = 6; + // The index of the field type in `Type.oneofs`, for message or enumeration + // types. The first type has index 1; zero means the type is not in the list. + int32 oneof_index = 7; + // Whether to use alternative packed wire representation. + bool packed = 8; + // The protocol buffer options. + repeated Option options = 9; + // The field JSON name. + string json_name = 10; + // The string value of the default value of this field. Proto2 syntax only. + string default_value = 11; +} + +// Enum type definition. +message Enum { + // Enum type name. + string name = 1; + // Enum value definitions. + repeated EnumValue enumvalue = 2; + // Protocol buffer options. + repeated Option options = 3; + // The source context. + SourceContext source_context = 4; + // The source syntax. + Syntax syntax = 5; +} + +// Enum value definition. +message EnumValue { + // Enum value name. + string name = 1; + // Enum value number. + int32 number = 2; + // Protocol buffer options. + repeated Option options = 3; +} + +// A protocol buffer option, which can be attached to a message, field, +// enumeration, etc. +message Option { + // The option's name. For protobuf built-in options (options defined in + // descriptor.proto), this is the short name. For example, `"map_entry"`. + // For custom options, it should be the fully-qualified name. For example, + // `"google.api.http"`. + string name = 1; + // The option's value packed in an Any message. If the value is a primitive, + // the corresponding wrapper type defined in google/protobuf/wrappers.proto + // should be used. If the value is an enum, it should be stored as an int32 + // value using the google.protobuf.Int32Value type. + Any value = 2; +} + +// The syntax in which a protocol buffer element is defined. +enum Syntax { + // Syntax `proto2`. + SYNTAX_PROTO2 = 0; + // Syntax `proto3`. + SYNTAX_PROTO3 = 1; +} diff --git a/proto/google/protobuf/wrappers.proto b/proto/google/protobuf/wrappers.proto new file mode 100644 index 0000000..d49dd53 --- /dev/null +++ b/proto/google/protobuf/wrappers.proto @@ -0,0 +1,123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Wrappers for primitive (non-message) types. These types are useful +// for embedding primitives in the `google.protobuf.Any` type and for places +// where we need to distinguish between the absence of a primitive +// typed field and its default value. +// +// These wrappers have no meaningful use within repeated fields as they lack +// the ability to detect presence on individual elements. +// These wrappers have no meaningful use within a map or a oneof since +// individual entries of a map or fields of a oneof can already detect presence. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/wrapperspb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "WrappersProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// Wrapper message for `double`. +// +// The JSON representation for `DoubleValue` is JSON number. +message DoubleValue { + // The double value. + double value = 1; +} + +// Wrapper message for `float`. +// +// The JSON representation for `FloatValue` is JSON number. +message FloatValue { + // The float value. + float value = 1; +} + +// Wrapper message for `int64`. +// +// The JSON representation for `Int64Value` is JSON string. +message Int64Value { + // The int64 value. + int64 value = 1; +} + +// Wrapper message for `uint64`. +// +// The JSON representation for `UInt64Value` is JSON string. +message UInt64Value { + // The uint64 value. + uint64 value = 1; +} + +// Wrapper message for `int32`. +// +// The JSON representation for `Int32Value` is JSON number. +message Int32Value { + // The int32 value. + int32 value = 1; +} + +// Wrapper message for `uint32`. +// +// The JSON representation for `UInt32Value` is JSON number. +message UInt32Value { + // The uint32 value. + uint32 value = 1; +} + +// Wrapper message for `bool`. +// +// The JSON representation for `BoolValue` is JSON `true` and `false`. +message BoolValue { + // The bool value. + bool value = 1; +} + +// Wrapper message for `string`. +// +// The JSON representation for `StringValue` is JSON string. +message StringValue { + // The string value. + string value = 1; +} + +// Wrapper message for `bytes`. +// +// The JSON representation for `BytesValue` is JSON string. +message BytesValue { + // The bytes value. + bytes value = 1; +} diff --git a/proto/google/pubsub/v1/BUILD b/proto/google/pubsub/v1/BUILD new file mode 100644 index 0000000..bf76daf --- /dev/null +++ b/proto/google/pubsub/v1/BUILD @@ -0,0 +1,16 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "pubsub_proto", + srcs = [ + "pubsub.proto", + "schema.proto", + ], + strip_import_prefix = "/proto/", + deps = [ + "//proto/google:com_google_protobuf", + "//proto/google:googleapis", + ], +) diff --git a/proto/google/pubsub/v1/pubsub.proto b/proto/google/pubsub/v1/pubsub.proto new file mode 100644 index 0000000..b1d6562 --- /dev/null +++ b/proto/google/pubsub/v1/pubsub.proto @@ -0,0 +1,1344 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.pubsub.v1; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/timestamp.proto"; +import "google/pubsub/v1/schema.proto"; + +option cc_enable_arenas = true; +option csharp_namespace = "Google.Cloud.PubSub.V1"; +option go_package = "google.golang.org/genproto/googleapis/pubsub/v1;pubsub"; +option java_multiple_files = true; +option java_outer_classname = "PubsubProto"; +option java_package = "com.google.pubsub.v1"; +option php_namespace = "Google\\Cloud\\PubSub\\V1"; +option ruby_package = "Google::Cloud::PubSub::V1"; + +// The service that an application uses to manipulate topics, and to send +// messages to a topic. +service Publisher { + option (google.api.default_host) = "pubsub.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/pubsub"; + + // Creates the given topic with the given name. See the [resource name rules] + // (https://cloud.google.com/pubsub/docs/admin#resource_names). + rpc CreateTopic(Topic) returns (Topic) { + option (google.api.http) = { + put: "/v1/{name=projects/*/topics/*}" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + + // Updates an existing topic. Note that certain properties of a + // topic are not modifiable. + rpc UpdateTopic(UpdateTopicRequest) returns (Topic) { + option (google.api.http) = { + patch: "/v1/{topic.name=projects/*/topics/*}" + body: "*" + }; + } + + // Adds one or more messages to the topic. Returns `NOT_FOUND` if the topic + // does not exist. + rpc Publish(PublishRequest) returns (PublishResponse) { + option (google.api.http) = { + post: "/v1/{topic=projects/*/topics/*}:publish" + body: "*" + }; + option (google.api.method_signature) = "topic,messages"; + } + + // Gets the configuration of a topic. + rpc GetTopic(GetTopicRequest) returns (Topic) { + option (google.api.http) = { + get: "/v1/{topic=projects/*/topics/*}" + }; + option (google.api.method_signature) = "topic"; + } + + // Lists matching topics. + rpc ListTopics(ListTopicsRequest) returns (ListTopicsResponse) { + option (google.api.http) = { + get: "/v1/{project=projects/*}/topics" + }; + option (google.api.method_signature) = "project"; + } + + // Lists the names of the attached subscriptions on this topic. + rpc ListTopicSubscriptions(ListTopicSubscriptionsRequest) + returns (ListTopicSubscriptionsResponse) { + option (google.api.http) = { + get: "/v1/{topic=projects/*/topics/*}/subscriptions" + }; + option (google.api.method_signature) = "topic"; + } + + // Lists the names of the snapshots on this topic. Snapshots are used in + // [Seek](https://cloud.google.com/pubsub/docs/replay-overview) operations, + // which allow you to manage message acknowledgments in bulk. That is, you can + // set the acknowledgment state of messages in an existing subscription to the + // state captured by a snapshot. + rpc ListTopicSnapshots(ListTopicSnapshotsRequest) + returns (ListTopicSnapshotsResponse) { + option (google.api.http) = { + get: "/v1/{topic=projects/*/topics/*}/snapshots" + }; + option (google.api.method_signature) = "topic"; + } + + // Deletes the topic with the given name. Returns `NOT_FOUND` if the topic + // does not exist. After a topic is deleted, a new topic may be created with + // the same name; this is an entirely new topic with none of the old + // configuration or subscriptions. Existing subscriptions to this topic are + // not deleted, but their `topic` field is set to `_deleted-topic_`. + rpc DeleteTopic(DeleteTopicRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{topic=projects/*/topics/*}" + }; + option (google.api.method_signature) = "topic"; + } + + // Detaches a subscription from this topic. All messages retained in the + // subscription are dropped. Subsequent `Pull` and `StreamingPull` requests + // will return FAILED_PRECONDITION. If the subscription is a push + // subscription, pushes to the endpoint will stop. + rpc DetachSubscription(DetachSubscriptionRequest) + returns (DetachSubscriptionResponse) { + option (google.api.http) = { + post: "/v1/{subscription=projects/*/subscriptions/*}:detach" + }; + } +} + +// A policy constraining the storage of messages published to the topic. +message MessageStoragePolicy { + // A list of IDs of GCP regions where messages that are published to the topic + // may be persisted in storage. Messages published by publishers running in + // non-allowed GCP regions (or running outside of GCP altogether) will be + // routed for storage in one of the allowed regions. An empty list means that + // no regions are allowed, and is not a valid configuration. + repeated string allowed_persistence_regions = 1; +} + +// Settings for validating messages published against a schema. +message SchemaSettings { + // Required. The name of the schema that messages published should be + // validated against. Format is `projects/{project}/schemas/{schema}`. The + // value of this field will be `_deleted-schema_` if the schema has been + // deleted. + string schema = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Schema" } + ]; + + // The encoding of messages validated against `schema`. + Encoding encoding = 2; +} + +// A topic resource. +message Topic { + option (google.api.resource) = { + type: "pubsub.googleapis.com/Topic" + pattern: "projects/{project}/topics/{topic}" + pattern: "_deleted-topic_" + }; + + // Required. The name of the topic. It must have the format + // `"projects/{project}/topics/{topic}"`. `{topic}` must start with a letter, + // and contain only letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`), + // underscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent + // signs (`%`). It must be between 3 and 255 characters in length, and it + // must not start with `"goog"`. + string name = 1 [(google.api.field_behavior) = REQUIRED]; + + // See [Creating and managing labels] + // (https://cloud.google.com/pubsub/docs/labels). + map labels = 2; + + // Policy constraining the set of Google Cloud Platform regions where messages + // published to the topic may be stored. If not present, then no constraints + // are in effect. + MessageStoragePolicy message_storage_policy = 3; + + // The resource name of the Cloud KMS CryptoKey to be used to protect access + // to messages published on this topic. + // + // The expected format is `projects/*/locations/*/keyRings/*/cryptoKeys/*`. + string kms_key_name = 5; + + // Settings for validating messages published against a schema. + SchemaSettings schema_settings = 6; + + // Reserved for future use. This field is set only in responses from the + // server; it is ignored if it is set in any requests. + bool satisfies_pzs = 7; + + // Indicates the minimum duration to retain a message after it is published to + // the topic. If this field is set, messages published to the topic in the + // last `message_retention_duration` are always available to subscribers. For + // instance, it allows any attached subscription to [seek to a + // timestamp](https://cloud.google.com/pubsub/docs/replay-overview#seek_to_a_time) + // that is up to `message_retention_duration` in the past. If this field is + // not set, message retention is controlled by settings on individual + // subscriptions. Cannot be more than 7 days or less than 10 minutes. + google.protobuf.Duration message_retention_duration = 8; +} + +// A message that is published by publishers and consumed by subscribers. The +// message must contain either a non-empty data field or at least one attribute. +// Note that client libraries represent this object differently +// depending on the language. See the corresponding [client library +// documentation](https://cloud.google.com/pubsub/docs/reference/libraries) for +// more information. See [quotas and limits] +// (https://cloud.google.com/pubsub/quotas) for more information about message +// limits. +message PubsubMessage { + // The message data field. If this field is empty, the message must contain + // at least one attribute. + bytes data = 1; + + // Attributes for this message. If this field is empty, the message must + // contain non-empty data. This can be used to filter messages on the + // subscription. + map attributes = 2; + + // ID of this message, assigned by the server when the message is published. + // Guaranteed to be unique within the topic. This value may be read by a + // subscriber that receives a `PubsubMessage` via a `Pull` call or a push + // delivery. It must not be populated by the publisher in a `Publish` call. + string message_id = 3; + + // The time at which the message was published, populated by the server when + // it receives the `Publish` call. It must not be populated by the + // publisher in a `Publish` call. + google.protobuf.Timestamp publish_time = 4; + + // If non-empty, identifies related messages for which publish order should be + // respected. If a `Subscription` has `enable_message_ordering` set to `true`, + // messages published with the same non-empty `ordering_key` value will be + // delivered to subscribers in the order in which they are received by the + // Pub/Sub system. All `PubsubMessage`s published in a given `PublishRequest` + // must specify the same `ordering_key` value. + string ordering_key = 5; +} + +// Request for the GetTopic method. +message GetTopicRequest { + // Required. The name of the topic to get. + // Format is `projects/{project}/topics/{topic}`. + string topic = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Topic" } + ]; +} + +// Request for the UpdateTopic method. +message UpdateTopicRequest { + // Required. The updated topic object. + Topic topic = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. Indicates which fields in the provided topic to update. Must be + // specified and non-empty. Note that if `update_mask` contains + // "message_storage_policy" but the `message_storage_policy` is not set in + // the `topic` provided above, then the updated value is determined by the + // policy configured at the project or organization level. + google.protobuf.FieldMask update_mask = 2 + [(google.api.field_behavior) = REQUIRED]; +} + +// Request for the Publish method. +message PublishRequest { + // Required. The messages in the request will be published on this topic. + // Format is `projects/{project}/topics/{topic}`. + string topic = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Topic" } + ]; + + // Required. The messages to publish. + repeated PubsubMessage messages = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// Response for the `Publish` method. +message PublishResponse { + // The server-assigned ID of each published message, in the same order as + // the messages in the request. IDs are guaranteed to be unique within + // the topic. + repeated string message_ids = 1; +} + +// Request for the `ListTopics` method. +message ListTopicsRequest { + // Required. The name of the project in which to list topics. + // Format is `projects/{project-id}`. + string project = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; + + // Maximum number of topics to return. + int32 page_size = 2; + + // The value returned by the last `ListTopicsResponse`; indicates that this is + // a continuation of a prior `ListTopics` call, and that the system should + // return the next page of data. + string page_token = 3; +} + +// Response for the `ListTopics` method. +message ListTopicsResponse { + // The resulting topics. + repeated Topic topics = 1; + + // If not empty, indicates that there may be more topics that match the + // request; this value should be passed in a new `ListTopicsRequest`. + string next_page_token = 2; +} + +// Request for the `ListTopicSubscriptions` method. +message ListTopicSubscriptionsRequest { + // Required. The name of the topic that subscriptions are attached to. + // Format is `projects/{project}/topics/{topic}`. + string topic = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Topic" } + ]; + + // Maximum number of subscription names to return. + int32 page_size = 2; + + // The value returned by the last `ListTopicSubscriptionsResponse`; indicates + // that this is a continuation of a prior `ListTopicSubscriptions` call, and + // that the system should return the next page of data. + string page_token = 3; +} + +// Response for the `ListTopicSubscriptions` method. +message ListTopicSubscriptionsResponse { + // The names of subscriptions attached to the topic specified in the request. + repeated string subscriptions = 1 [(google.api.resource_reference) = { + type: "pubsub.googleapis.com/Subscription" + }]; + + // If not empty, indicates that there may be more subscriptions that match + // the request; this value should be passed in a new + // `ListTopicSubscriptionsRequest` to get more subscriptions. + string next_page_token = 2; +} + +// Request for the `ListTopicSnapshots` method. +message ListTopicSnapshotsRequest { + // Required. The name of the topic that snapshots are attached to. + // Format is `projects/{project}/topics/{topic}`. + string topic = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Topic" } + ]; + + // Maximum number of snapshot names to return. + int32 page_size = 2; + + // The value returned by the last `ListTopicSnapshotsResponse`; indicates + // that this is a continuation of a prior `ListTopicSnapshots` call, and + // that the system should return the next page of data. + string page_token = 3; +} + +// Response for the `ListTopicSnapshots` method. +message ListTopicSnapshotsResponse { + // The names of the snapshots that match the request. + repeated string snapshots = 1; + + // If not empty, indicates that there may be more snapshots that match + // the request; this value should be passed in a new + // `ListTopicSnapshotsRequest` to get more snapshots. + string next_page_token = 2; +} + +// Request for the `DeleteTopic` method. +message DeleteTopicRequest { + // Required. Name of the topic to delete. + // Format is `projects/{project}/topics/{topic}`. + string topic = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Topic" } + ]; +} + +// Request for the DetachSubscription method. +message DetachSubscriptionRequest { + // Required. The subscription to detach. + // Format is `projects/{project}/subscriptions/{subscription}`. + string subscription = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "pubsub.googleapis.com/Subscription" + } + ]; +} + +// Response for the DetachSubscription method. +// Reserved for future use. +message DetachSubscriptionResponse {} + +// The service that an application uses to manipulate subscriptions and to +// consume messages from a subscription via the `Pull` method or by +// establishing a bi-directional stream using the `StreamingPull` method. +service Subscriber { + option (google.api.default_host) = "pubsub.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/pubsub"; + + // Creates a subscription to a given topic. See the [resource name rules] + // (https://cloud.google.com/pubsub/docs/admin#resource_names). + // If the subscription already exists, returns `ALREADY_EXISTS`. + // If the corresponding topic doesn't exist, returns `NOT_FOUND`. + // + // If the name is not provided in the request, the server will assign a random + // name for this subscription on the same project as the topic, conforming + // to the [resource name format] + // (https://cloud.google.com/pubsub/docs/admin#resource_names). The generated + // name is populated in the returned Subscription object. Note that for REST + // API requests, you must specify a name in the request. + rpc CreateSubscription(Subscription) returns (Subscription) { + option (google.api.http) = { + put: "/v1/{name=projects/*/subscriptions/*}" + body: "*" + }; + option (google.api.method_signature) = + "name,topic,push_config,ack_deadline_seconds"; + } + + // Gets the configuration details of a subscription. + rpc GetSubscription(GetSubscriptionRequest) returns (Subscription) { + option (google.api.http) = { + get: "/v1/{subscription=projects/*/subscriptions/*}" + }; + option (google.api.method_signature) = "subscription"; + } + + // Updates an existing subscription. Note that certain properties of a + // subscription, such as its topic, are not modifiable. + rpc UpdateSubscription(UpdateSubscriptionRequest) returns (Subscription) { + option (google.api.http) = { + patch: "/v1/{subscription.name=projects/*/subscriptions/*}" + body: "*" + }; + } + + // Lists matching subscriptions. + rpc ListSubscriptions(ListSubscriptionsRequest) + returns (ListSubscriptionsResponse) { + option (google.api.http) = { + get: "/v1/{project=projects/*}/subscriptions" + }; + option (google.api.method_signature) = "project"; + } + + // Deletes an existing subscription. All messages retained in the subscription + // are immediately dropped. Calls to `Pull` after deletion will return + // `NOT_FOUND`. After a subscription is deleted, a new one may be created with + // the same name, but the new one has no association with the old + // subscription or its topic unless the same topic is specified. + rpc DeleteSubscription(DeleteSubscriptionRequest) + returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{subscription=projects/*/subscriptions/*}" + }; + option (google.api.method_signature) = "subscription"; + } + + // Modifies the ack deadline for a specific message. This method is useful + // to indicate that more time is needed to process a message by the + // subscriber, or to make the message available for redelivery if the + // processing was interrupted. Note that this does not modify the + // subscription-level `ackDeadlineSeconds` used for subsequent messages. + rpc ModifyAckDeadline(ModifyAckDeadlineRequest) + returns (google.protobuf.Empty) { + option (google.api.http) = { + post: "/v1/{subscription=projects/*/subscriptions/*}:modifyAckDeadline" + body: "*" + }; + option (google.api.method_signature) = + "subscription,ack_ids,ack_deadline_seconds"; + } + + // Acknowledges the messages associated with the `ack_ids` in the + // `AcknowledgeRequest`. The Pub/Sub system can remove the relevant messages + // from the subscription. + // + // Acknowledging a message whose ack deadline has expired may succeed, + // but such a message may be redelivered later. Acknowledging a message more + // than once will not result in an error. + rpc Acknowledge(AcknowledgeRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + post: "/v1/{subscription=projects/*/subscriptions/*}:acknowledge" + body: "*" + }; + option (google.api.method_signature) = "subscription,ack_ids"; + } + + // Pulls messages from the server. The server may return `UNAVAILABLE` if + // there are too many concurrent pull requests pending for the given + // subscription. + rpc Pull(PullRequest) returns (PullResponse) { + option (google.api.http) = { + post: "/v1/{subscription=projects/*/subscriptions/*}:pull" + body: "*" + }; + option (google.api.method_signature) = + "subscription,return_immediately,max_messages"; + option (google.api.method_signature) = "subscription,max_messages"; + } + + // Establishes a stream with the server, which sends messages down to the + // client. The client streams acknowledgements and ack deadline modifications + // back to the server. The server will close the stream and return the status + // on any error. The server may close the stream with status `UNAVAILABLE` to + // reassign server-side resources, in which case, the client should + // re-establish the stream. Flow control can be achieved by configuring the + // underlying RPC channel. + rpc StreamingPull(stream StreamingPullRequest) + returns (stream StreamingPullResponse) {} + + // Modifies the `PushConfig` for a specified subscription. + // + // This may be used to change a push subscription to a pull one (signified by + // an empty `PushConfig`) or vice versa, or change the endpoint URL and other + // attributes of a push subscription. Messages will accumulate for delivery + // continuously through the call regardless of changes to the `PushConfig`. + rpc ModifyPushConfig(ModifyPushConfigRequest) + returns (google.protobuf.Empty) { + option (google.api.http) = { + post: "/v1/{subscription=projects/*/subscriptions/*}:modifyPushConfig" + body: "*" + }; + option (google.api.method_signature) = "subscription,push_config"; + } + + // Gets the configuration details of a snapshot. Snapshots are used in + // Seek + // operations, which allow you to manage message acknowledgments in bulk. That + // is, you can set the acknowledgment state of messages in an existing + // subscription to the state captured by a snapshot. + rpc GetSnapshot(GetSnapshotRequest) returns (Snapshot) { + option (google.api.http) = { + get: "/v1/{snapshot=projects/*/snapshots/*}" + }; + option (google.api.method_signature) = "snapshot"; + } + + // Lists the existing snapshots. Snapshots are used in [Seek]( + // https://cloud.google.com/pubsub/docs/replay-overview) operations, which + // allow you to manage message acknowledgments in bulk. That is, you can set + // the acknowledgment state of messages in an existing subscription to the + // state captured by a snapshot. + rpc ListSnapshots(ListSnapshotsRequest) returns (ListSnapshotsResponse) { + option (google.api.http) = { + get: "/v1/{project=projects/*}/snapshots" + }; + option (google.api.method_signature) = "project"; + } + + // Creates a snapshot from the requested subscription. Snapshots are used in + // [Seek](https://cloud.google.com/pubsub/docs/replay-overview) operations, + // which allow you to manage message acknowledgments in bulk. That is, you can + // set the acknowledgment state of messages in an existing subscription to the + // state captured by a snapshot. + // If the snapshot already exists, returns `ALREADY_EXISTS`. + // If the requested subscription doesn't exist, returns `NOT_FOUND`. + // If the backlog in the subscription is too old -- and the resulting snapshot + // would expire in less than 1 hour -- then `FAILED_PRECONDITION` is returned. + // See also the `Snapshot.expire_time` field. If the name is not provided in + // the request, the server will assign a random + // name for this snapshot on the same project as the subscription, conforming + // to the [resource name format] + // (https://cloud.google.com/pubsub/docs/admin#resource_names). The + // generated name is populated in the returned Snapshot object. Note that for + // REST API requests, you must specify a name in the request. + rpc CreateSnapshot(CreateSnapshotRequest) returns (Snapshot) { + option (google.api.http) = { + put: "/v1/{name=projects/*/snapshots/*}" + body: "*" + }; + option (google.api.method_signature) = "name,subscription"; + } + + // Updates an existing snapshot. Snapshots are used in + // Seek + // operations, which allow + // you to manage message acknowledgments in bulk. That is, you can set the + // acknowledgment state of messages in an existing subscription to the state + // captured by a snapshot. + rpc UpdateSnapshot(UpdateSnapshotRequest) returns (Snapshot) { + option (google.api.http) = { + patch: "/v1/{snapshot.name=projects/*/snapshots/*}" + body: "*" + }; + } + + // Removes an existing snapshot. Snapshots are used in [Seek] + // (https://cloud.google.com/pubsub/docs/replay-overview) operations, which + // allow you to manage message acknowledgments in bulk. That is, you can set + // the acknowledgment state of messages in an existing subscription to the + // state captured by a snapshot. + // When the snapshot is deleted, all messages retained in the snapshot + // are immediately dropped. After a snapshot is deleted, a new one may be + // created with the same name, but the new one has no association with the old + // snapshot or its subscription, unless the same subscription is specified. + rpc DeleteSnapshot(DeleteSnapshotRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{snapshot=projects/*/snapshots/*}" + }; + option (google.api.method_signature) = "snapshot"; + } + + // Seeks an existing subscription to a point in time or to a given snapshot, + // whichever is provided in the request. Snapshots are used in [Seek] + // (https://cloud.google.com/pubsub/docs/replay-overview) operations, which + // allow you to manage message acknowledgments in bulk. That is, you can set + // the acknowledgment state of messages in an existing subscription to the + // state captured by a snapshot. Note that both the subscription and the + // snapshot must be on the same topic. + rpc Seek(SeekRequest) returns (SeekResponse) { + option (google.api.http) = { + post: "/v1/{subscription=projects/*/subscriptions/*}:seek" + body: "*" + }; + } +} + +// A subscription resource. +message Subscription { + option (google.api.resource) = { + type: "pubsub.googleapis.com/Subscription" + pattern: "projects/{project}/subscriptions/{subscription}" + }; + + // Required. The name of the subscription. It must have the format + // `"projects/{project}/subscriptions/{subscription}"`. `{subscription}` must + // start with a letter, and contain only letters (`[A-Za-z]`), numbers + // (`[0-9]`), dashes (`-`), underscores (`_`), periods (`.`), tildes (`~`), + // plus (`+`) or percent signs (`%`). It must be between 3 and 255 characters + // in length, and it must not start with `"goog"`. + string name = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. The name of the topic from which this subscription is receiving + // messages. Format is `projects/{project}/topics/{topic}`. The value of this + // field will be `_deleted-topic_` if the topic has been deleted. + string topic = 2 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Topic" } + ]; + + // If push delivery is used with this subscription, this field is + // used to configure it. An empty `pushConfig` signifies that the subscriber + // will pull and ack messages using API methods. + PushConfig push_config = 4; + + // The approximate amount of time (on a best-effort basis) Pub/Sub waits for + // the subscriber to acknowledge receipt before resending the message. In the + // interval after the message is delivered and before it is acknowledged, it + // is considered to be outstanding. During that time period, the + // message will not be redelivered (on a best-effort basis). + // + // For pull subscriptions, this value is used as the initial value for the ack + // deadline. To override this value for a given message, call + // `ModifyAckDeadline` with the corresponding `ack_id` if using + // non-streaming pull or send the `ack_id` in a + // `StreamingModifyAckDeadlineRequest` if using streaming pull. + // The minimum custom deadline you can specify is 10 seconds. + // The maximum custom deadline you can specify is 600 seconds (10 minutes). + // If this parameter is 0, a default value of 10 seconds is used. + // + // For push delivery, this value is also used to set the request timeout for + // the call to the push endpoint. + // + // If the subscriber never acknowledges the message, the Pub/Sub + // system will eventually redeliver the message. + int32 ack_deadline_seconds = 5; + + // Indicates whether to retain acknowledged messages. If true, then + // messages are not expunged from the subscription's backlog, even if they are + // acknowledged, until they fall out of the `message_retention_duration` + // window. This must be true if you would like to [`Seek` to a timestamp] + // (https://cloud.google.com/pubsub/docs/replay-overview#seek_to_a_time) in + // the past to replay previously-acknowledged messages. + bool retain_acked_messages = 7; + + // How long to retain unacknowledged messages in the subscription's backlog, + // from the moment a message is published. + // If `retain_acked_messages` is true, then this also configures the retention + // of acknowledged messages, and thus configures how far back in time a `Seek` + // can be done. Defaults to 7 days. Cannot be more than 7 days or less than 10 + // minutes. + google.protobuf.Duration message_retention_duration = 8; + + // See Creating and + // managing labels. + map labels = 9; + + // If true, messages published with the same `ordering_key` in `PubsubMessage` + // will be delivered to the subscribers in the order in which they + // are received by the Pub/Sub system. Otherwise, they may be delivered in + // any order. + bool enable_message_ordering = 10; + + // A policy that specifies the conditions for this subscription's expiration. + // A subscription is considered active as long as any connected subscriber is + // successfully consuming messages from the subscription or is issuing + // operations on the subscription. If `expiration_policy` is not set, a + // *default policy* with `ttl` of 31 days will be used. The minimum allowed + // value for `expiration_policy.ttl` is 1 day. + ExpirationPolicy expiration_policy = 11; + + // An expression written in the Pub/Sub [filter + // language](https://cloud.google.com/pubsub/docs/filtering). If non-empty, + // then only `PubsubMessage`s whose `attributes` field matches the filter are + // delivered on this subscription. If empty, then no messages are filtered + // out. + string filter = 12; + + // A policy that specifies the conditions for dead lettering messages in + // this subscription. If dead_letter_policy is not set, dead lettering + // is disabled. + // + // The Cloud Pub/Sub service account associated with this subscriptions's + // parent project (i.e., + // service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com) must have + // permission to Acknowledge() messages on this subscription. + DeadLetterPolicy dead_letter_policy = 13; + + // A policy that specifies how Pub/Sub retries message delivery for this + // subscription. + // + // If not set, the default retry policy is applied. This generally implies + // that messages will be retried as soon as possible for healthy subscribers. + // RetryPolicy will be triggered on NACKs or acknowledgement deadline + // exceeded events for a given message. + RetryPolicy retry_policy = 14; + + // Indicates whether the subscription is detached from its topic. Detached + // subscriptions don't receive messages from their topic and don't retain any + // backlog. `Pull` and `StreamingPull` requests will return + // FAILED_PRECONDITION. If the subscription is a push subscription, pushes to + // the endpoint will not be made. + bool detached = 15; + + // Output only. Indicates the minimum duration for which a message is retained + // after it is published to the subscription's topic. If this field is set, + // messages published to the subscription's topic in the last + // `topic_message_retention_duration` are always available to subscribers. See + // the `message_retention_duration` field in `Topic`. This field is set only + // in responses from the server; it is ignored if it is set in any requests. + google.protobuf.Duration topic_message_retention_duration = 17 + [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// A policy that specifies how Cloud Pub/Sub retries message delivery. +// +// Retry delay will be exponential based on provided minimum and maximum +// backoffs. https://en.wikipedia.org/wiki/Exponential_backoff. +// +// RetryPolicy will be triggered on NACKs or acknowledgement deadline exceeded +// events for a given message. +// +// Retry Policy is implemented on a best effort basis. At times, the delay +// between consecutive deliveries may not match the configuration. That is, +// delay can be more or less than configured backoff. +message RetryPolicy { + // The minimum delay between consecutive deliveries of a given message. + // Value should be between 0 and 600 seconds. Defaults to 10 seconds. + google.protobuf.Duration minimum_backoff = 1; + + // The maximum delay between consecutive deliveries of a given message. + // Value should be between 0 and 600 seconds. Defaults to 600 seconds. + google.protobuf.Duration maximum_backoff = 2; +} + +// Dead lettering is done on a best effort basis. The same message might be +// dead lettered multiple times. +// +// If validation on any of the fields fails at subscription creation/updation, +// the create/update subscription request will fail. +message DeadLetterPolicy { + // The name of the topic to which dead letter messages should be published. + // Format is `projects/{project}/topics/{topic}`.The Cloud Pub/Sub service + // account associated with the enclosing subscription's parent project (i.e., + // service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com) must have + // permission to Publish() to this topic. + // + // The operation will fail if the topic does not exist. + // Users should ensure that there is a subscription attached to this topic + // since messages published to a topic with no subscriptions are lost. + string dead_letter_topic = 1; + + // The maximum number of delivery attempts for any message. The value must be + // between 5 and 100. + // + // The number of delivery attempts is defined as 1 + (the sum of number of + // NACKs and number of times the acknowledgement deadline has been exceeded + // for the message). + // + // A NACK is any call to ModifyAckDeadline with a 0 deadline. Note that + // client libraries may automatically extend ack_deadlines. + // + // This field will be honored on a best effort basis. + // + // If this parameter is 0, a default value of 5 is used. + int32 max_delivery_attempts = 2; +} + +// A policy that specifies the conditions for resource expiration (i.e., +// automatic resource deletion). +message ExpirationPolicy { + // Specifies the "time-to-live" duration for an associated resource. The + // resource expires if it is not active for a period of `ttl`. The definition + // of "activity" depends on the type of the associated resource. The minimum + // and maximum allowed values for `ttl` depend on the type of the associated + // resource, as well. If `ttl` is not set, the associated resource never + // expires. + google.protobuf.Duration ttl = 1; +} + +// Configuration for a push delivery endpoint. +message PushConfig { + // Contains information needed for generating an + // [OpenID Connect + // token](https://developers.google.com/identity/protocols/OpenIDConnect). + message OidcToken { + // [Service account + // email](https://cloud.google.com/iam/docs/service-accounts) + // to be used for generating the OIDC token. The caller (for + // CreateSubscription, UpdateSubscription, and ModifyPushConfig RPCs) must + // have the iam.serviceAccounts.actAs permission for the service account. + string service_account_email = 1; + + // Audience to be used when generating OIDC token. The audience claim + // identifies the recipients that the JWT is intended for. The audience + // value is a single case-sensitive string. Having multiple values (array) + // for the audience field is not supported. More info about the OIDC JWT + // token audience here: https://tools.ietf.org/html/rfc7519#section-4.1.3 + // Note: if not specified, the Push endpoint URL will be used. + string audience = 2; + } + + // A URL locating the endpoint to which messages should be pushed. + // For example, a Webhook endpoint might use `https://example.com/push`. + string push_endpoint = 1; + + // Endpoint configuration attributes that can be used to control different + // aspects of the message delivery. + // + // The only currently supported attribute is `x-goog-version`, which you can + // use to change the format of the pushed message. This attribute + // indicates the version of the data expected by the endpoint. This + // controls the shape of the pushed message (i.e., its fields and metadata). + // + // If not present during the `CreateSubscription` call, it will default to + // the version of the Pub/Sub API used to make such call. If not present in a + // `ModifyPushConfig` call, its value will not be changed. `GetSubscription` + // calls will always return a valid version, even if the subscription was + // created without this attribute. + // + // The only supported values for the `x-goog-version` attribute are: + // + // * `v1beta1`: uses the push format defined in the v1beta1 Pub/Sub API. + // * `v1` or `v1beta2`: uses the push format defined in the v1 Pub/Sub API. + // + // For example: + //
attributes { "x-goog-version": "v1" } 
+ map attributes = 2; + + // An authentication method used by push endpoints to verify the source of + // push requests. This can be used with push endpoints that are private by + // default to allow requests only from the Cloud Pub/Sub system, for example. + // This field is optional and should be set only by users interested in + // authenticated push. + oneof authentication_method { + // If specified, Pub/Sub will generate and attach an OIDC JWT token as an + // `Authorization` header in the HTTP request for every pushed message. + OidcToken oidc_token = 3; + } +} + +// A message and its corresponding acknowledgment ID. +message ReceivedMessage { + // This ID can be used to acknowledge the received message. + string ack_id = 1; + + // The message. + PubsubMessage message = 2; + + // The approximate number of times that Cloud Pub/Sub has attempted to deliver + // the associated message to a subscriber. + // + // More precisely, this is 1 + (number of NACKs) + + // (number of ack_deadline exceeds) for this message. + // + // A NACK is any call to ModifyAckDeadline with a 0 deadline. An ack_deadline + // exceeds event is whenever a message is not acknowledged within + // ack_deadline. Note that ack_deadline is initially + // Subscription.ackDeadlineSeconds, but may get extended automatically by + // the client library. + // + // Upon the first delivery of a given message, `delivery_attempt` will have a + // value of 1. The value is calculated at best effort and is approximate. + // + // If a DeadLetterPolicy is not set on the subscription, this will be 0. + int32 delivery_attempt = 3; +} + +// Request for the GetSubscription method. +message GetSubscriptionRequest { + // Required. The name of the subscription to get. + // Format is `projects/{project}/subscriptions/{sub}`. + string subscription = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "pubsub.googleapis.com/Subscription" + } + ]; +} + +// Request for the UpdateSubscription method. +message UpdateSubscriptionRequest { + // Required. The updated subscription object. + Subscription subscription = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. Indicates which fields in the provided subscription to update. + // Must be specified and non-empty. + google.protobuf.FieldMask update_mask = 2 + [(google.api.field_behavior) = REQUIRED]; +} + +// Request for the `ListSubscriptions` method. +message ListSubscriptionsRequest { + // Required. The name of the project in which to list subscriptions. + // Format is `projects/{project-id}`. + string project = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; + + // Maximum number of subscriptions to return. + int32 page_size = 2; + + // The value returned by the last `ListSubscriptionsResponse`; indicates that + // this is a continuation of a prior `ListSubscriptions` call, and that the + // system should return the next page of data. + string page_token = 3; +} + +// Response for the `ListSubscriptions` method. +message ListSubscriptionsResponse { + // The subscriptions that match the request. + repeated Subscription subscriptions = 1; + + // If not empty, indicates that there may be more subscriptions that match + // the request; this value should be passed in a new + // `ListSubscriptionsRequest` to get more subscriptions. + string next_page_token = 2; +} + +// Request for the DeleteSubscription method. +message DeleteSubscriptionRequest { + // Required. The subscription to delete. + // Format is `projects/{project}/subscriptions/{sub}`. + string subscription = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "pubsub.googleapis.com/Subscription" + } + ]; +} + +// Request for the ModifyPushConfig method. +message ModifyPushConfigRequest { + // Required. The name of the subscription. + // Format is `projects/{project}/subscriptions/{sub}`. + string subscription = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "pubsub.googleapis.com/Subscription" + } + ]; + + // Required. The push configuration for future deliveries. + // + // An empty `pushConfig` indicates that the Pub/Sub system should + // stop pushing messages from the given subscription and allow + // messages to be pulled and acknowledged - effectively pausing + // the subscription if `Pull` or `StreamingPull` is not called. + PushConfig push_config = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// Request for the `Pull` method. +message PullRequest { + // Required. The subscription from which messages should be pulled. + // Format is `projects/{project}/subscriptions/{sub}`. + string subscription = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "pubsub.googleapis.com/Subscription" + } + ]; + + // Optional. If this field set to true, the system will respond immediately + // even if it there are no messages available to return in the `Pull` + // response. Otherwise, the system may wait (for a bounded amount of time) + // until at least one message is available, rather than returning no messages. + // Warning: setting this field to `true` is discouraged because it adversely + // impacts the performance of `Pull` operations. We recommend that users do + // not set this field. + bool return_immediately = 2 + [deprecated = true, (google.api.field_behavior) = OPTIONAL]; + + // Required. The maximum number of messages to return for this request. Must + // be a positive integer. The Pub/Sub system may return fewer than the number + // specified. + int32 max_messages = 3 [(google.api.field_behavior) = REQUIRED]; +} + +// Response for the `Pull` method. +message PullResponse { + // Received Pub/Sub messages. The list will be empty if there are no more + // messages available in the backlog. For JSON, the response can be entirely + // empty. The Pub/Sub system may return fewer than the `maxMessages` requested + // even if there are more messages available in the backlog. + repeated ReceivedMessage received_messages = 1; +} + +// Request for the ModifyAckDeadline method. +message ModifyAckDeadlineRequest { + // Required. The name of the subscription. + // Format is `projects/{project}/subscriptions/{sub}`. + string subscription = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "pubsub.googleapis.com/Subscription" + } + ]; + + // Required. List of acknowledgment IDs. + repeated string ack_ids = 4 [(google.api.field_behavior) = REQUIRED]; + + // Required. The new ack deadline with respect to the time this request was + // sent to the Pub/Sub system. For example, if the value is 10, the new ack + // deadline will expire 10 seconds after the `ModifyAckDeadline` call was + // made. Specifying zero might immediately make the message available for + // delivery to another subscriber client. This typically results in an + // increase in the rate of message redeliveries (that is, duplicates). + // The minimum deadline you can specify is 0 seconds. + // The maximum deadline you can specify is 600 seconds (10 minutes). + int32 ack_deadline_seconds = 3 [(google.api.field_behavior) = REQUIRED]; +} + +// Request for the Acknowledge method. +message AcknowledgeRequest { + // Required. The subscription whose message is being acknowledged. + // Format is `projects/{project}/subscriptions/{sub}`. + string subscription = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "pubsub.googleapis.com/Subscription" + } + ]; + + // Required. The acknowledgment ID for the messages being acknowledged that + // was returned by the Pub/Sub system in the `Pull` response. Must not be + // empty. + repeated string ack_ids = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// Request for the `StreamingPull` streaming RPC method. This request is used to +// establish the initial stream as well as to stream acknowledgements and ack +// deadline modifications from the client to the server. +message StreamingPullRequest { + // Required. The subscription for which to initialize the new stream. This + // must be provided in the first request on the stream, and must not be set in + // subsequent requests from client to server. + // Format is `projects/{project}/subscriptions/{sub}`. + string subscription = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "pubsub.googleapis.com/Subscription" + } + ]; + + // List of acknowledgement IDs for acknowledging previously received messages + // (received on this stream or a different stream). If an ack ID has expired, + // the corresponding message may be redelivered later. Acknowledging a message + // more than once will not result in an error. If the acknowledgement ID is + // malformed, the stream will be aborted with status `INVALID_ARGUMENT`. + repeated string ack_ids = 2; + + // The list of new ack deadlines for the IDs listed in + // `modify_deadline_ack_ids`. The size of this list must be the same as the + // size of `modify_deadline_ack_ids`. If it differs the stream will be aborted + // with `INVALID_ARGUMENT`. Each element in this list is applied to the + // element in the same position in `modify_deadline_ack_ids`. The new ack + // deadline is with respect to the time this request was sent to the Pub/Sub + // system. Must be >= 0. For example, if the value is 10, the new ack deadline + // will expire 10 seconds after this request is received. If the value is 0, + // the message is immediately made available for another streaming or + // non-streaming pull request. If the value is < 0 (an error), the stream will + // be aborted with status `INVALID_ARGUMENT`. + repeated int32 modify_deadline_seconds = 3; + + // List of acknowledgement IDs whose deadline will be modified based on the + // corresponding element in `modify_deadline_seconds`. This field can be used + // to indicate that more time is needed to process a message by the + // subscriber, or to make the message available for redelivery if the + // processing was interrupted. + repeated string modify_deadline_ack_ids = 4; + + // Required. The ack deadline to use for the stream. This must be provided in + // the first request on the stream, but it can also be updated on subsequent + // requests from client to server. The minimum deadline you can specify is 10 + // seconds. The maximum deadline you can specify is 600 seconds (10 minutes). + int32 stream_ack_deadline_seconds = 5 + [(google.api.field_behavior) = REQUIRED]; + + // A unique identifier that is used to distinguish client instances from each + // other. Only needs to be provided on the initial request. When a stream + // disconnects and reconnects for the same stream, the client_id should be set + // to the same value so that state associated with the old stream can be + // transferred to the new stream. The same client_id should not be used for + // different client instances. + string client_id = 6; + + // Flow control settings for the maximum number of outstanding messages. When + // there are `max_outstanding_messages` or more currently sent to the + // streaming pull client that have not yet been acked or nacked, the server + // stops sending more messages. The sending of messages resumes once the + // number of outstanding messages is less than this value. If the value is + // <= 0, there is no limit to the number of outstanding messages. This + // property can only be set on the initial StreamingPullRequest. If it is set + // on a subsequent request, the stream will be aborted with status + // `INVALID_ARGUMENT`. + int64 max_outstanding_messages = 7; + + // Flow control settings for the maximum number of outstanding bytes. When + // there are `max_outstanding_bytes` or more worth of messages currently sent + // to the streaming pull client that have not yet been acked or nacked, the + // server will stop sending more messages. The sending of messages resumes + // once the number of outstanding bytes is less than this value. If the value + // is <= 0, there is no limit to the number of outstanding bytes. This + // property can only be set on the initial StreamingPullRequest. If it is set + // on a subsequent request, the stream will be aborted with status + // `INVALID_ARGUMENT`. + int64 max_outstanding_bytes = 8; +} + +// Response for the `StreamingPull` method. This response is used to stream +// messages from the server to the client. +message StreamingPullResponse { + // Subscription properties sent as part of the response. + message SubscriptionProperties { + // True iff message ordering is enabled for this subscription. + bool message_ordering_enabled = 2; + } + + // Received Pub/Sub messages. This will not be empty. + repeated ReceivedMessage received_messages = 1; + + // Properties associated with this subscription. + SubscriptionProperties subscription_properties = 4; +} + +// Request for the `CreateSnapshot` method. +message CreateSnapshotRequest { + // Required. User-provided name for this snapshot. If the name is not provided + // in the request, the server will assign a random name for this snapshot on + // the same project as the subscription. Note that for REST API requests, you + // must specify a name. See the resource + // name rules. Format is `projects/{project}/snapshots/{snap}`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Snapshot" } + ]; + + // Required. The subscription whose backlog the snapshot retains. + // Specifically, the created snapshot is guaranteed to retain: + // (a) The existing backlog on the subscription. More precisely, this is + // defined as the messages in the subscription's backlog that are + // unacknowledged upon the successful completion of the + // `CreateSnapshot` request; as well as: + // (b) Any messages published to the subscription's topic following the + // successful completion of the CreateSnapshot request. + // Format is `projects/{project}/subscriptions/{sub}`. + string subscription = 2 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "pubsub.googleapis.com/Subscription" + } + ]; + + // See Creating and + // managing labels. + map labels = 3; +} + +// Request for the UpdateSnapshot method. +message UpdateSnapshotRequest { + // Required. The updated snapshot object. + Snapshot snapshot = 1 [(google.api.field_behavior) = REQUIRED]; + + // Required. Indicates which fields in the provided snapshot to update. + // Must be specified and non-empty. + google.protobuf.FieldMask update_mask = 2 + [(google.api.field_behavior) = REQUIRED]; +} + +// A snapshot resource. Snapshots are used in +// [Seek](https://cloud.google.com/pubsub/docs/replay-overview) +// operations, which allow you to manage message acknowledgments in bulk. That +// is, you can set the acknowledgment state of messages in an existing +// subscription to the state captured by a snapshot. +message Snapshot { + option (google.api.resource) = { + type: "pubsub.googleapis.com/Snapshot" + pattern: "projects/{project}/snapshots/{snapshot}" + }; + + // The name of the snapshot. + string name = 1; + + // The name of the topic from which this snapshot is retaining messages. + string topic = 2 [ + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Topic" } + ]; + + // The snapshot is guaranteed to exist up until this time. + // A newly-created snapshot expires no later than 7 days from the time of its + // creation. Its exact lifetime is determined at creation by the existing + // backlog in the source subscription. Specifically, the lifetime of the + // snapshot is `7 days - (age of oldest unacked message in the subscription)`. + // For example, consider a subscription whose oldest unacked message is 3 days + // old. If a snapshot is created from this subscription, the snapshot -- which + // will always capture this 3-day-old backlog as long as the snapshot + // exists -- will expire in 4 days. The service will refuse to create a + // snapshot that would expire in less than 1 hour after creation. + google.protobuf.Timestamp expire_time = 3; + + // See [Creating and managing labels] + // (https://cloud.google.com/pubsub/docs/labels). + map labels = 4; +} + +// Request for the GetSnapshot method. +message GetSnapshotRequest { + // Required. The name of the snapshot to get. + // Format is `projects/{project}/snapshots/{snap}`. + string snapshot = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Snapshot" } + ]; +} + +// Request for the `ListSnapshots` method. +message ListSnapshotsRequest { + // Required. The name of the project in which to list snapshots. + // Format is `projects/{project-id}`. + string project = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; + + // Maximum number of snapshots to return. + int32 page_size = 2; + + // The value returned by the last `ListSnapshotsResponse`; indicates that this + // is a continuation of a prior `ListSnapshots` call, and that the system + // should return the next page of data. + string page_token = 3; +} + +// Response for the `ListSnapshots` method. +message ListSnapshotsResponse { + // The resulting snapshots. + repeated Snapshot snapshots = 1; + + // If not empty, indicates that there may be more snapshot that match the + // request; this value should be passed in a new `ListSnapshotsRequest`. + string next_page_token = 2; +} + +// Request for the `DeleteSnapshot` method. +message DeleteSnapshotRequest { + // Required. The name of the snapshot to delete. + // Format is `projects/{project}/snapshots/{snap}`. + string snapshot = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Snapshot" } + ]; +} + +// Request for the `Seek` method. +message SeekRequest { + // Required. The subscription to affect. + string subscription = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "pubsub.googleapis.com/Subscription" + } + ]; + + oneof target { + // The time to seek to. + // Messages retained in the subscription that were published before this + // time are marked as acknowledged, and messages retained in the + // subscription that were published after this time are marked as + // unacknowledged. Note that this operation affects only those messages + // retained in the subscription (configured by the combination of + // `message_retention_duration` and `retain_acked_messages`). For example, + // if `time` corresponds to a point before the message retention + // window (or to a point before the system's notion of the subscription + // creation time), only retained messages will be marked as unacknowledged, + // and already-expunged messages will not be restored. + google.protobuf.Timestamp time = 2; + + // The snapshot to seek to. The snapshot's topic must be the same as that of + // the provided subscription. + // Format is `projects/{project}/snapshots/{snap}`. + string snapshot = 3 [(google.api.resource_reference) = { + type: "pubsub.googleapis.com/Snapshot" + }]; + } +} + +// Response for the `Seek` method (this response is empty). +message SeekResponse {} diff --git a/proto/google/pubsub/v1/schema.proto b/proto/google/pubsub/v1/schema.proto new file mode 100644 index 0000000..3cdf10d --- /dev/null +++ b/proto/google/pubsub/v1/schema.proto @@ -0,0 +1,288 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.pubsub.v1; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/empty.proto"; + +option cc_enable_arenas = true; +option csharp_namespace = "Google.Cloud.PubSub.V1"; +option go_package = "google.golang.org/genproto/googleapis/pubsub/v1;pubsub"; +option java_multiple_files = true; +option java_outer_classname = "SchemaProto"; +option java_package = "com.google.pubsub.v1"; +option php_namespace = "Google\\Cloud\\PubSub\\V1"; +option ruby_package = "Google::Cloud::PubSub::V1"; + +// Service for doing schema-related operations. +service SchemaService { + option (google.api.default_host) = "pubsub.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/pubsub"; + + // Creates a schema. + rpc CreateSchema(CreateSchemaRequest) returns (Schema) { + option (google.api.http) = { + post: "/v1/{parent=projects/*}/schemas" + body: "schema" + }; + option (google.api.method_signature) = "parent,schema,schema_id"; + } + + // Gets a schema. + rpc GetSchema(GetSchemaRequest) returns (Schema) { + option (google.api.http) = { + get: "/v1/{name=projects/*/schemas/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Lists schemas in a project. + rpc ListSchemas(ListSchemasRequest) returns (ListSchemasResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*}/schemas" + }; + option (google.api.method_signature) = "parent"; + } + + // Deletes a schema. + rpc DeleteSchema(DeleteSchemaRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/schemas/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Validates a schema. + rpc ValidateSchema(ValidateSchemaRequest) returns (ValidateSchemaResponse) { + option (google.api.http) = { + post: "/v1/{parent=projects/*}/schemas:validate" + body: "*" + }; + option (google.api.method_signature) = "parent,schema"; + } + + // Validates a message against a schema. + rpc ValidateMessage(ValidateMessageRequest) + returns (ValidateMessageResponse) { + option (google.api.http) = { + post: "/v1/{parent=projects/*}/schemas:validateMessage" + body: "*" + }; + } +} + +// A schema resource. +message Schema { + option (google.api.resource) = { + type: "pubsub.googleapis.com/Schema" + pattern: "projects/{project}/schemas/{schema}" + }; + + // Possible schema definition types. + enum Type { + // Default value. This value is unused. + TYPE_UNSPECIFIED = 0; + + // A Protocol Buffer schema definition. + PROTOCOL_BUFFER = 1; + + // An Avro schema definition. + AVRO = 2; + } + + // Required. Name of the schema. + // Format is `projects/{project}/schemas/{schema}`. + string name = 1 [(google.api.field_behavior) = REQUIRED]; + + // The type of the schema definition. + Type type = 2; + + // The definition of the schema. This should contain a string representing + // the full definition of the schema that is a valid schema definition of + // the type specified in `type`. + string definition = 3; +} + +// Request for the CreateSchema method. +message CreateSchemaRequest { + // Required. The name of the project in which to create the schema. + // Format is `projects/{project-id}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + child_type: "pubsub.googleapis.com/Schema" + } + ]; + + // Required. The schema object to create. + // + // This schema's `name` parameter is ignored. The schema object returned + // by CreateSchema will have a `name` made using the given `parent` and + // `schema_id`. + Schema schema = 2 [(google.api.field_behavior) = REQUIRED]; + + // The ID to use for the schema, which will become the final component of + // the schema's resource name. + // + // See https://cloud.google.com/pubsub/docs/admin#resource_names for resource + // name constraints. + string schema_id = 3; +} + +// View of Schema object fields to be returned by GetSchema and ListSchemas. +enum SchemaView { + // The default / unset value. + // The API will default to the BASIC view. + SCHEMA_VIEW_UNSPECIFIED = 0; + + // Include the name and type of the schema, but not the definition. + BASIC = 1; + + // Include all Schema object fields. + FULL = 2; +} + +// Request for the GetSchema method. +message GetSchemaRequest { + // Required. The name of the schema to get. + // Format is `projects/{project}/schemas/{schema}`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Schema" } + ]; + + // The set of fields to return in the response. If not set, returns a Schema + // with `name` and `type`, but not `definition`. Set to `FULL` to retrieve all + // fields. + SchemaView view = 2; +} + +// Request for the `ListSchemas` method. +message ListSchemasRequest { + // Required. The name of the project in which to list schemas. + // Format is `projects/{project-id}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; + + // The set of Schema fields to return in the response. If not set, returns + // Schemas with `name` and `type`, but not `definition`. Set to `FULL` to + // retrieve all fields. + SchemaView view = 2; + + // Maximum number of schemas to return. + int32 page_size = 3; + + // The value returned by the last `ListSchemasResponse`; indicates that + // this is a continuation of a prior `ListSchemas` call, and that the + // system should return the next page of data. + string page_token = 4; +} + +// Response for the `ListSchemas` method. +message ListSchemasResponse { + // The resulting schemas. + repeated Schema schemas = 1; + + // If not empty, indicates that there may be more schemas that match the + // request; this value should be passed in a new `ListSchemasRequest`. + string next_page_token = 2; +} + +// Request for the `DeleteSchema` method. +message DeleteSchemaRequest { + // Required. Name of the schema to delete. + // Format is `projects/{project}/schemas/{schema}`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Schema" } + ]; +} + +// Request for the `ValidateSchema` method. +message ValidateSchemaRequest { + // Required. The name of the project in which to validate schemas. + // Format is `projects/{project-id}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; + + // Required. The schema object to validate. + Schema schema = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// Response for the `ValidateSchema` method. +// Empty for now. +message ValidateSchemaResponse {} + +// Request for the `ValidateMessage` method. +message ValidateMessageRequest { + // Required. The name of the project in which to validate schemas. + // Format is `projects/{project-id}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; + + oneof schema_spec { + // Name of the schema against which to validate. + // + // Format is `projects/{project}/schemas/{schema}`. + string name = 2 [ + (google.api.resource_reference) = { type: "pubsub.googleapis.com/Schema" } + ]; + + // Ad-hoc schema against which to validate + Schema schema = 3; + } + + // Message to validate against the provided `schema_spec`. + bytes message = 4; + + // The encoding expected for messages + Encoding encoding = 5; +} + +// Response for the `ValidateMessage` method. +// Empty for now. +message ValidateMessageResponse {} + +// Possible encoding types for messages. +enum Encoding { + // Unspecified + ENCODING_UNSPECIFIED = 0; + + // JSON encoding + JSON = 1; + + // Binary encoding, as defined by the schema type. For some schema types, + // binary encoding may not be available. + BINARY = 2; +} diff --git a/proto/google/rpc/code.proto b/proto/google/rpc/code.proto new file mode 100644 index 0000000..98ae0ac --- /dev/null +++ b/proto/google/rpc/code.proto @@ -0,0 +1,186 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.rpc; + +option go_package = "google.golang.org/genproto/googleapis/rpc/code;code"; +option java_multiple_files = true; +option java_outer_classname = "CodeProto"; +option java_package = "com.google.rpc"; +option objc_class_prefix = "RPC"; + +// The canonical error codes for gRPC APIs. +// +// +// Sometimes multiple error codes may apply. Services should return +// the most specific error code that applies. For example, prefer +// `OUT_OF_RANGE` over `FAILED_PRECONDITION` if both codes apply. +// Similarly prefer `NOT_FOUND` or `ALREADY_EXISTS` over `FAILED_PRECONDITION`. +enum Code { + // Not an error; returned on success + // + // HTTP Mapping: 200 OK + OK = 0; + + // The operation was cancelled, typically by the caller. + // + // HTTP Mapping: 499 Client Closed Request + CANCELLED = 1; + + // Unknown error. For example, this error may be returned when + // a `Status` value received from another address space belongs to + // an error space that is not known in this address space. Also + // errors raised by APIs that do not return enough error information + // may be converted to this error. + // + // HTTP Mapping: 500 Internal Server Error + UNKNOWN = 2; + + // The client specified an invalid argument. Note that this differs + // from `FAILED_PRECONDITION`. `INVALID_ARGUMENT` indicates arguments + // that are problematic regardless of the state of the system + // (e.g., a malformed file name). + // + // HTTP Mapping: 400 Bad Request + INVALID_ARGUMENT = 3; + + // The deadline expired before the operation could complete. For operations + // that change the state of the system, this error may be returned + // even if the operation has completed successfully. For example, a + // successful response from a server could have been delayed long + // enough for the deadline to expire. + // + // HTTP Mapping: 504 Gateway Timeout + DEADLINE_EXCEEDED = 4; + + // Some requested entity (e.g., file or directory) was not found. + // + // Note to server developers: if a request is denied for an entire class + // of users, such as gradual feature rollout or undocumented whitelist, + // `NOT_FOUND` may be used. If a request is denied for some users within + // a class of users, such as user-based access control, `PERMISSION_DENIED` + // must be used. + // + // HTTP Mapping: 404 Not Found + NOT_FOUND = 5; + + // The entity that a client attempted to create (e.g., file or directory) + // already exists. + // + // HTTP Mapping: 409 Conflict + ALREADY_EXISTS = 6; + + // The caller does not have permission to execute the specified + // operation. `PERMISSION_DENIED` must not be used for rejections + // caused by exhausting some resource (use `RESOURCE_EXHAUSTED` + // instead for those errors). `PERMISSION_DENIED` must not be + // used if the caller can not be identified (use `UNAUTHENTICATED` + // instead for those errors). This error code does not imply the + // request is valid or the requested entity exists or satisfies + // other pre-conditions. + // + // HTTP Mapping: 403 Forbidden + PERMISSION_DENIED = 7; + + // The request does not have valid authentication credentials for the + // operation. + // + // HTTP Mapping: 401 Unauthorized + UNAUTHENTICATED = 16; + + // Some resource has been exhausted, perhaps a per-user quota, or + // perhaps the entire file system is out of space. + // + // HTTP Mapping: 429 Too Many Requests + RESOURCE_EXHAUSTED = 8; + + // The operation was rejected because the system is not in a state + // required for the operation's execution. For example, the directory + // to be deleted is non-empty, an rmdir operation is applied to + // a non-directory, etc. + // + // Service implementors can use the following guidelines to decide + // between `FAILED_PRECONDITION`, `ABORTED`, and `UNAVAILABLE`: + // (a) Use `UNAVAILABLE` if the client can retry just the failing call. + // (b) Use `ABORTED` if the client should retry at a higher level + // (e.g., when a client-specified test-and-set fails, indicating the + // client should restart a read-modify-write sequence). + // (c) Use `FAILED_PRECONDITION` if the client should not retry until + // the system state has been explicitly fixed. E.g., if an "rmdir" + // fails because the directory is non-empty, `FAILED_PRECONDITION` + // should be returned since the client should not retry unless + // the files are deleted from the directory. + // + // HTTP Mapping: 400 Bad Request + FAILED_PRECONDITION = 9; + + // The operation was aborted, typically due to a concurrency issue such as + // a sequencer check failure or transaction abort. + // + // See the guidelines above for deciding between `FAILED_PRECONDITION`, + // `ABORTED`, and `UNAVAILABLE`. + // + // HTTP Mapping: 409 Conflict + ABORTED = 10; + + // The operation was attempted past the valid range. E.g., seeking or + // reading past end-of-file. + // + // Unlike `INVALID_ARGUMENT`, this error indicates a problem that may + // be fixed if the system state changes. For example, a 32-bit file + // system will generate `INVALID_ARGUMENT` if asked to read at an + // offset that is not in the range [0,2^32-1], but it will generate + // `OUT_OF_RANGE` if asked to read from an offset past the current + // file size. + // + // There is a fair bit of overlap between `FAILED_PRECONDITION` and + // `OUT_OF_RANGE`. We recommend using `OUT_OF_RANGE` (the more specific + // error) when it applies so that callers who are iterating through + // a space can easily look for an `OUT_OF_RANGE` error to detect when + // they are done. + // + // HTTP Mapping: 400 Bad Request + OUT_OF_RANGE = 11; + + // The operation is not implemented or is not supported/enabled in this + // service. + // + // HTTP Mapping: 501 Not Implemented + UNIMPLEMENTED = 12; + + // Internal errors. This means that some invariants expected by the + // underlying system have been broken. This error code is reserved + // for serious errors. + // + // HTTP Mapping: 500 Internal Server Error + INTERNAL = 13; + + // The service is currently unavailable. This is most likely a + // transient condition, which can be corrected by retrying with + // a backoff. Note that it is not always safe to retry + // non-idempotent operations. + // + // See the guidelines above for deciding between `FAILED_PRECONDITION`, + // `ABORTED`, and `UNAVAILABLE`. + // + // HTTP Mapping: 503 Service Unavailable + UNAVAILABLE = 14; + + // Unrecoverable data loss or corruption. + // + // HTTP Mapping: 500 Internal Server Error + DATA_LOSS = 15; +} diff --git a/proto/google/rpc/status.proto b/proto/google/rpc/status.proto new file mode 100644 index 0000000..3b1f7a9 --- /dev/null +++ b/proto/google/rpc/status.proto @@ -0,0 +1,47 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.rpc; + +import "google/protobuf/any.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/rpc/status;status"; +option java_multiple_files = true; +option java_outer_classname = "StatusProto"; +option java_package = "com.google.rpc"; +option objc_class_prefix = "RPC"; + +// The `Status` type defines a logical error model that is suitable for +// different programming environments, including REST APIs and RPC APIs. It is +// used by [gRPC](https://github.com/grpc). Each `Status` message contains +// three pieces of data: error code, error message, and error details. +// +// You can find out more about this error model and how to work with it in the +// [API Design Guide](https://cloud.google.com/apis/design/errors). +message Status { + // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + int32 code = 1; + + // A developer-facing error message, which should be in English. Any + // user-facing error message should be localized and sent in the + // [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. + string message = 2; + + // A list of messages that carry the error details. There is a common set of + // message types for APIs to use. + repeated google.protobuf.Any details = 3; +} diff --git a/proto/google/type/BUILD b/proto/google/type/BUILD new file mode 100644 index 0000000..bf6b2f2 --- /dev/null +++ b/proto/google/type/BUILD @@ -0,0 +1,9 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "expr_proto", + srcs = ["expr.proto"], + strip_import_prefix = "/proto/", +) diff --git a/proto/google/type/date.proto b/proto/google/type/date.proto new file mode 100644 index 0000000..e4e730e --- /dev/null +++ b/proto/google/type/date.proto @@ -0,0 +1,52 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/date;date"; +option java_multiple_files = true; +option java_outer_classname = "DateProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a whole or partial calendar date, such as a birthday. The time of +// day and time zone are either specified elsewhere or are insignificant. The +// date is relative to the Gregorian Calendar. This can represent one of the +// following: +// +// * A full date, with non-zero year, month, and day values +// * A month and day value, with a zero year, such as an anniversary +// * A year on its own, with zero month and day values +// * A year and month value, with a zero day, such as a credit card expiration +// date +// +// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and +// `google.protobuf.Timestamp`. +message Date { + // Year of the date. Must be from 1 to 9999, or 0 to specify a date without + // a year. + int32 year = 1; + + // Month of a year. Must be from 1 to 12, or 0 to specify a year without a + // month and day. + int32 month = 2; + + // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 + // to specify a year by itself or a year and month where the day isn't + // significant. + int32 day = 3; +} diff --git a/proto/google/type/expr.proto b/proto/google/type/expr.proto new file mode 100644 index 0000000..af0778c --- /dev/null +++ b/proto/google/type/expr.proto @@ -0,0 +1,73 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; +option java_multiple_files = true; +option java_outer_classname = "ExprProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a textual expression in the Common Expression Language (CEL) +// syntax. CEL is a C-like expression language. The syntax and semantics of CEL +// are documented at https://github.com/google/cel-spec. +// +// Example (Comparison): +// +// title: "Summary size limit" +// description: "Determines if a summary is less than 100 chars" +// expression: "document.summary.size() < 100" +// +// Example (Equality): +// +// title: "Requestor is owner" +// description: "Determines if requestor is the document owner" +// expression: "document.owner == request.auth.claims.email" +// +// Example (Logic): +// +// title: "Public documents" +// description: "Determine whether the document should be publicly visible" +// expression: "document.type != 'private' && document.type != 'internal'" +// +// Example (Data Manipulation): +// +// title: "Notification string" +// description: "Create a notification string with a timestamp." +// expression: "'New message received at ' + string(document.create_time)" +// +// The exact variables and functions that may be referenced within an expression +// are determined by the service that evaluates it. See the service +// documentation for additional information. +message Expr { + // Textual representation of an expression in Common Expression Language + // syntax. + string expression = 1; + + // Optional. Title for the expression, i.e. a short string describing + // its purpose. This can be used e.g. in UIs which allow to enter the + // expression. + string title = 2; + + // Optional. Description of the expression. This is a longer text which + // describes the expression, e.g. when hovered over it in a UI. + string description = 3; + + // Optional. String indicating the location of the expression for error + // reporting, e.g. a file name and a position in the file. + string location = 4; +} diff --git a/proto/osprey/rpc/actions/v1/BUILD b/proto/osprey/rpc/actions/v1/BUILD new file mode 100644 index 0000000..cd1ec93 --- /dev/null +++ b/proto/osprey/rpc/actions/v1/BUILD @@ -0,0 +1,30 @@ +load("@rules_java//java:defs.bzl", "java_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +proto_library( + name = "actions_proto", + srcs = [ + "action.proto", + "action_types.proto", + "application.proto", + "captcha.proto", + "channel.proto", + "guild.proto", + "invite.proto", + "metadata.proto", + "user.proto", + ], + strip_import_prefix = "/proto", + visibility = ["//visibility:public"], + deps = [ + "//proto/discord_protos/discord_authentication/v1:session_proto", + "//proto/discord_protos/users/v1:user_proto", + "//proto/google:com_google_protobuf", + ], +) + +java_proto_library( + name = "actions_proto_java", + visibility = ["//visibility:public"], + deps = [":actions_proto"], +) diff --git a/proto/osprey/rpc/actions/v1/action.proto b/proto/osprey/rpc/actions/v1/action.proto new file mode 100644 index 0000000..9031f97 --- /dev/null +++ b/proto/osprey/rpc/actions/v1/action.proto @@ -0,0 +1,182 @@ +syntax = "proto3"; + +package discord_smite_rpc.actions.v1; + +import "discord_protos/discord_authentication/v1/session.proto"; +import "discord_smite_rpc/actions/v1/action_types.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +message Action { + // Snowflake id of action message + // Currently actions from discord_authentication use a pseudo snowflake to reduce dependencies. + fixed64 id = 1; + + // This oneof denotes the type of action this message is. + // Each field name in this this oneof should match the name of an action that Smite expects. + oneof data { + AuthSessionCreated auth_session_created = 2; + AuthSessionModified auth_session_modified = 3; + UserFlinkApplicationScored user_flink_application_scored = 4; + GuildFlinkApplicationScored guild_flink_application_scored = 5; + GuildCreated guild_created = 6; + UserBatchMLScore user_batch_ml_score = 7; + UserBlocked user_blocked = 8; + GuildJoined guild_joined = 9; + SmiteAutoClusteringUser smite_auto_clustering_user = 10; + SmiteAutoClusteringKey smite_auto_clustering_key = 11; + HashMatcherIconMatched hash_matcher_icon_matched = 12; + SafetyVisualPrediction safety_visual_prediction = 13; + } +} + +message AuthSessionCreated { + UserAuth user = 1; + UserSession user_session = 2; +} + +message AuthSessionModified { + UserAuth user = 1; + UserSession user_session = 2; +} + +message UserFlinkApplicationScored { + google.protobuf.Timestamp timestamp = 1; + google.protobuf.UInt64Value source_event_id = 2; + google.protobuf.Timestamp source_event_timestamp = 3; + UserFlink user = 4; + string application_name = 5; + string application_version = 6; + double score = 7; +} + +message GuildFlinkApplicationScored { + google.protobuf.Timestamp timestamp = 1; + google.protobuf.UInt64Value source_event_id = 2; + google.protobuf.Timestamp source_event_timestamp = 3; + uint64 guild_id = 4; + string application_name = 5; + string application_version = 6; + double score = 7; +} + +message OrchestrationJob { + string tool_name = 1; // Orchestration tool used (e.g., 'airflow', 'argo', 'dagster') + string job_id = 2; // Orchestration tool unique job id (e.g., Airflow run_id, Argo workflow ID, Dagster run ID) +} + +message UserBatchMLScore { + google.protobuf.Timestamp timestamp = 1; + UserBatchML user = 2; + OrchestrationJob orchestration_job = 3; + string application_name = 4; + string application_version = 5; + double score = 6; + BatchMLCluster cluster = 7; + BatchMLSourceEvent source_event = 8; +} + +message SmiteAutoClusteringUser { + google.protobuf.Timestamp timestamp = 1; + uint64 user_id = 2; + string pipeline_name = 3; + string cluster_id = 4; +} + +message SmiteAutoClusteringKey { + google.protobuf.Timestamp timestamp = 1; + string key = 2; + string value = 3; + string pipeline_name = 4; + string cluster_id = 5; + uint64 cluster_size = 6; + string entity_type = 7; +} + +message HashMatcherIconMatched { + google.protobuf.Timestamp timestamp = 1; + string content_id = 2; + string hash_algorithm = 3; + double distance = 4; + string seed_id = 5; + string qdrant_collection_name = 6; + string md5 = 7; + string policy = 8; + string seed_gcs_uri = 9; + uint64 guild_id = 10; + string seed_metadata = 11; + UserHashMatcher user = 12; +} + +message SafetyVisualPrediction { + google.protobuf.Timestamp timestamp = 1; + uint64 guild_id = 2; + uint64 channel_id = 3; + uint64 message_id = 4; + uint64 attachment_id = 5; + uint64 user_id = 6; + uint64 webhook_id = 7; + string prediction_class = 8; + float score = 9; + bool decision = 10; + float decision_threshold = 11; + string md5_hash = 12; + bool from_bot = 13; + string scored_class = 14; + string source = 15; +} + +// Cluster ID and metadata for ML clustering batch jobs +message BatchMLCluster { + // cluster_id is a uuid + string cluster_id = 1; + + // cluster_features is a list of strings in the format ['featureA:valueA', 'featureB:valueB'] + repeated string cluster_features = 2; + + // whether we want to apply a label to future actions that match this cluster's cluster_features, + // or just act on the individual entities identified in this cluster on this batch run + bool apply_label = 3; +} + +message UserSession { + bytes id_hash = 1; + discord_protos.discord_authentication.v1.UserSessionData.Version session_version = 2; + google.protobuf.Timestamp creation_time = 3; + google.protobuf.Timestamp expiration_time = 4; + google.protobuf.Timestamp approx_last_used_time = 5; + discord_protos.discord_authentication.v1.UserSessionData.ClientInfo client_info = 6; + // Previous client info of the session if a change was detected + discord_protos.discord_authentication.v1.UserSessionData.ClientInfo prev_client_info = 7; + google.protobuf.Timestamp prev_approx_last_used_time = 8; +} + +// A subset of user object fields that are populated by the Auth service +message UserAuth { + fixed64 id = 1; + bool mfa_enabled = 2; + bool bot = 3; + int64 flags = 4; +} + +// User field for Flink events +message UserFlink { + fixed64 id = 1; +} + +// User field for Batch ML jobs +message UserBatchML { + fixed64 id = 1; +} + +// User field for HashMatcher events +message UserHashMatcher { + fixed64 id = 1; +} + +// Source event for a UserBatchMLScore +message BatchMLSourceEvent { + google.protobuf.StringValue source_event_id = 1; + google.protobuf.Timestamp source_event_timestamp = 2; + google.protobuf.UInt64Value source_guild_id = 3; +} diff --git a/proto/osprey/rpc/actions/v1/action_types.proto b/proto/osprey/rpc/actions/v1/action_types.proto new file mode 100644 index 0000000..723ee27 --- /dev/null +++ b/proto/osprey/rpc/actions/v1/action_types.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +package discord_smite_rpc.actions.v1; + +import "discord_smite_rpc/actions/v1/application.proto"; +import "discord_smite_rpc/actions/v1/captcha.proto"; +import "discord_smite_rpc/actions/v1/channel.proto"; +import "discord_smite_rpc/actions/v1/guild.proto"; +import "discord_smite_rpc/actions/v1/invite.proto"; +import "discord_smite_rpc/actions/v1/metadata.proto"; +import "discord_smite_rpc/actions/v1/user.proto"; + +message GuildCreated { + User user = 1; + Guild guild = 2; + HttpRequestData http_request = 3; + Properties properties = 4; +} + +message UserBlocked { + User user = 1; + User target_user = 2; + Guild guild = 3; + + GuildMember guild_member = 4; + GuildMember target_guild_member = 5; + + bool did_solve_captcha = 6; + int32 num_mutual_guilds = 7; + int32 source = 8; + int32 user_guild_count = 9; + + HttpRequestData http_request = 10; + Properties properties = 11; +} + +message GuildJoined { + User user = 1; + User inviter = 2; + Guild guild = 3; + GuildInvite invite = 4; + Channel channel = 5; + Application application = 6; + + CaptchaResponse captcha_response = 7; + bool did_solve_captcha = 8; + + HttpRequestData http_request = 9; + Properties properties = 10; +} diff --git a/proto/osprey/rpc/actions/v1/application.proto b/proto/osprey/rpc/actions/v1/application.proto new file mode 100644 index 0000000..48073e8 --- /dev/null +++ b/proto/osprey/rpc/actions/v1/application.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package discord_smite_rpc.actions.v1; + +import "discord_smite_rpc/actions/v1/user.proto"; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +message Application { + fixed64 id = 1; + fixed64 bot_id = 2; + User bot = 3; + string name = 4; + fixed64 eventual_owner_user_id = 5; + google.protobuf.StringValue team_id = 6; +} diff --git a/proto/osprey/rpc/actions/v1/captcha.proto b/proto/osprey/rpc/actions/v1/captcha.proto new file mode 100644 index 0000000..4b58450 --- /dev/null +++ b/proto/osprey/rpc/actions/v1/captcha.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package discord_smite_rpc.actions.v1; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +message CaptchaResponse { + string captcha_service = 1; + bool success = 2; + google.protobuf.Timestamp challenge_ts = 3; + repeated string error_codes = 4; + map sigs = 5; + google.protobuf.StringValue solve_token = 6; + + // HCaptcha and RECaptcha + google.protobuf.StringValue hostname = 7; + + // HCaptcha-only fields + google.protobuf.FloatValue score = 8; + repeated string score_reason = 9; + google.protobuf.StringValue sitekey = 10; + google.protobuf.StringValue scoped_uid_0 = 11; + google.protobuf.StringValue scoped_uid_1 = 12; +} diff --git a/proto/osprey/rpc/actions/v1/channel.proto b/proto/osprey/rpc/actions/v1/channel.proto new file mode 100644 index 0000000..dc6a3fd --- /dev/null +++ b/proto/osprey/rpc/actions/v1/channel.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package discord_smite_rpc.actions.v1; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +message PrivateChannelRecipient { + fixed64 id = 1; +} + +message Channel { + fixed64 id = 1; + string name = 2; + int32 type = 3; + bool is_nsfw = 4; + int32 message_count = 5; + repeated PrivateChannelRecipient recipients = 6; +} diff --git a/proto/osprey/rpc/actions/v1/guild.proto b/proto/osprey/rpc/actions/v1/guild.proto new file mode 100644 index 0000000..453a0b8 --- /dev/null +++ b/proto/osprey/rpc/actions/v1/guild.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package discord_smite_rpc.actions.v1; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +// Smite representation of a guild object +message Guild { + fixed64 id = 1; + GuildOwner owner = 2; + string name = 3; + google.protobuf.StringValue description = 4; + google.protobuf.StringValue icon_hash = 5; + bool is_nsfw = 6; + int32 nsfw_level = 7; + int32 member_count = 8; + int32 premium_subscription_count = 9; + repeated string features = 10; + bool has_public_stage = 11; + repeated string channel_names = 12; + repeated string role_names = 13; + int32 verification_level = 14; + GuildTemplate template = 15; +} + +message GuildOwner { + fixed64 id = 1; +} + +message GuildTemplate { + string code = 1; + string name = 2; + google.protobuf.StringValue description = 3; + fixed64 creator_id = 4; + google.protobuf.Timestamp created_at = 5; + google.protobuf.Timestamp updated_at = 6; + fixed64 source_guild_id = 7; +} + +message GuildMember { + fixed64 guild_id = 1; + google.protobuf.Timestamp joined_at = 2; + // Maybe this should be a 64 + fixed64 permissions = 3; + fixed64 user_id = 4; +} diff --git a/proto/osprey/rpc/actions/v1/invite.proto b/proto/osprey/rpc/actions/v1/invite.proto new file mode 100644 index 0000000..ff5bf0b --- /dev/null +++ b/proto/osprey/rpc/actions/v1/invite.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package discord_smite_rpc.actions.v1; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +message GuildInvite { + string code = 1; + int32 max_age = 2; + google.protobuf.Timestamp created_at = 3; + int32 max_uses = 4; + bool temporary = 5; + bool custom = 6; +} diff --git a/proto/osprey/rpc/actions/v1/metadata.proto b/proto/osprey/rpc/actions/v1/metadata.proto new file mode 100644 index 0000000..814aa3a --- /dev/null +++ b/proto/osprey/rpc/actions/v1/metadata.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +package discord_smite_rpc.actions.v1; + +import "google/protobuf/wrappers.proto"; + +message HttpRequestData { + map headers = 1; + + string url = 2; + google.protobuf.StringValue endpoint = 3; + google.protobuf.StringValue query_string = 4; + string method = 5; + google.protobuf.Int32Value api_version = 6; + + google.protobuf.StringValue remote_addr = 7; + google.protobuf.BoolValue remote_addr_is_whitelisted = 8; + +} + +message Properties { + google.protobuf.StringValue os = 1; + google.protobuf.StringValue os_version = 2; + google.protobuf.StringValue browser = 3; + google.protobuf.StringValue browser_version = 4; + google.protobuf.StringValue browser_user_agent = 5; + google.protobuf.StringValue device = 6; + google.protobuf.StringValue referrer = 7; + google.protobuf.StringValue referring_domain = 8; + google.protobuf.StringValue release_channel = 9; + google.protobuf.StringValue client_version = 10; + google.protobuf.Int64Value client_build_number = 11; + google.protobuf.StringValue location = 12; + google.protobuf.StringValue system_locale = 13; + google.protobuf.StringValue chosen_locale = 14; + google.protobuf.StringValue device_vendor_id = 15; + google.protobuf.StringValue cfduid = 16; + bool cfduid_signed = 17; +} diff --git a/proto/osprey/rpc/actions/v1/user.proto b/proto/osprey/rpc/actions/v1/user.proto new file mode 100644 index 0000000..e454bc8 --- /dev/null +++ b/proto/osprey/rpc/actions/v1/user.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package discord_smite_rpc.actions.v1; + +import "discord_protos/users/v1/user.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +message User { + reserved 10; + + fixed64 id = 1; + bool mfa_enabled = 2; + bool bot = 3; + string username = 4; + string discriminator = 5; + google.protobuf.StringValue email = 6; + bool email_verified = 7; + google.protobuf.StringValue phone = 8; + google.protobuf.StringValue avatar = 9; + string bio = 11; + google.protobuf.StringValue ip = 12; + bool ip_is_whitelisted = 13; + google.protobuf.UInt32Value premium_type = 14; + google.protobuf.Timestamp premium_until = 15; + fixed64 flags = 16; + fixed64 public_flags = 17; + int32 suspicious_activity_flags = 18; + google.protobuf.StringValue required_action = 19; + bool is_unique_username = 20; + google.protobuf.StringValue global_name = 21; + // Used by guild automation GUILD_JOINED event + google.protobuf.StringValue hub_email = 22; + discord_protos.users.v1.UserAvatarDecoration avatar_decoration_data = 23; +}