From 56bad5d44eb6c9e21b64a62bdda513f6fc9bc571 Mon Sep 17 00:00:00 2001 From: Xiaotong Chen Date: Tue, 29 Jul 2025 19:01:59 +0800 Subject: [PATCH 1/2] dashinfer vlm: add tensorrt support for qwen2.5vl --- .gitignore | 1 + multimodal/README.md | 2 +- .../dashinfer_vlm/visual_embedding/DFN_vit.py | 40 +- .../visual_embedding/DFN_vit_2_5.py | 497 ++++++++++++++++++ .../dashinfer_vlm/visual_embedding/utils.py | 21 + .../dashinfer_vlm/vl_inference/__init__.py | 3 +- .../vl_inference/utils/__init__.py | 2 +- .../vl_inference/utils/model_loader.py | 6 +- .../vl_inference/utils/trt/onnx_to_plan.py | 51 +- .../vl_inference/utils/trt/vit_process.py | 29 +- multimodal/tests/benchmark_openai_api.py | 8 +- multimodal/tests/test.jpg | Bin 0 -> 50031 bytes .../tests/test_openai_chat_completion.py | 20 +- 13 files changed, 575 insertions(+), 105 deletions(-) create mode 100644 multimodal/dashinfer_vlm/visual_embedding/DFN_vit_2_5.py create mode 100644 multimodal/dashinfer_vlm/visual_embedding/utils.py create mode 100644 multimodal/tests/test.jpg diff --git a/.gitignore b/.gitignore index 5753fddae..ac6826c87 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ third_party/from_source/openssl/* log* *.csv *.as* +*.egg-info diff --git a/multimodal/README.md b/multimodal/README.md index a91b07a2d..4eb702760 100644 --- a/multimodal/README.md +++ b/multimodal/README.md @@ -8,7 +8,7 @@ DashInfer VLMs is a toolkit to support Vision Language Models (VLMs) inference b ## Supported Models - Qwen2-VL 2B/7B/72B -- Qwen2.5-VL 2B/7B/72B (Only support transformers vit engine) +- Qwen2.5-VL 3B/7B/32B/72B ## Architecture ![alt text](resource/dashinfer-vlm-arch.png) diff --git a/multimodal/dashinfer_vlm/visual_embedding/DFN_vit.py b/multimodal/dashinfer_vlm/visual_embedding/DFN_vit.py index 71cd20f32..5deb7535c 100644 --- a/multimodal/dashinfer_vlm/visual_embedding/DFN_vit.py +++ b/multimodal/dashinfer_vlm/visual_embedding/DFN_vit.py @@ -21,32 +21,11 @@ from transformers.models.qwen2_vl.modeling_qwen2_vl import ( Qwen2VisionTransformerPretrainedModel, ) +from .utils import default_weight_loader -# from .model_loader import default_weight_loader dtype = "fp32" -def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: - """Default weight loader.""" - try: - if param.numel() == 1 and loaded_weight.numel() == 1: - # Sometimes scalar values aren't considered tensors with shapes - # so if both param and loaded_weight are a scalar, - # "broadcast" instead of copy - param.data.fill_(loaded_weight.item()) - else: - assert param.size() == loaded_weight.size(), ( - f"Attempted to load weight ({loaded_weight.size()}) " - f"into parameter ({param.size()})" - ) - - param.data.copy_(loaded_weight) - except Exception: - # NOTE: This exception is added for the purpose of setting breakpoint to - # debug weight loading issues. - raise - - def quick_gelu(x: torch.Tensor, inplace: bool = False) -> torch.Tensor: return x * torch.sigmoid(1.702 * x) @@ -400,25 +379,8 @@ def forward(self, x, cu_seqlens, rotary_pos_emb, b) -> torch.Tensor: x = x + self.mlp(self.norm2(x)) return x - -# class Qwen2VisionTransformer(nn.Module): class Qwen2VisionTransformer(Qwen2VisionTransformerPretrainedModel): def __init__(self, config): - # img_size: int = 378, - # patch_size: int = 14, - # temporal_patch_size: int = 2, - # spatial_merge_size: int = 2, - # in_chans: int = 3, - # hidden_size: int = 1000, - # embed_dim: int = 768, - # depth: int = 12, - # num_heads: int = 16, - # mlp_ratio: float = 4.0, - # norm_layer: nn.Module = partial(LayerNorm, eps=1e-6), - # use_flash_attention: bool = False, - # *args, - # **kwargs, - # ) -> None: super().__init__(config) self.spatial_merge_size = config.spatial_merge_size diff --git a/multimodal/dashinfer_vlm/visual_embedding/DFN_vit_2_5.py b/multimodal/dashinfer_vlm/visual_embedding/DFN_vit_2_5.py new file mode 100644 index 000000000..84796dfc5 --- /dev/null +++ b/multimodal/dashinfer_vlm/visual_embedding/DFN_vit_2_5.py @@ -0,0 +1,497 @@ +# Copyright (c) Alibaba Cloud. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import math +from functools import partial +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import LayerNorm +import torch.utils.checkpoint +from flash_attn.flash_attn_interface import flash_attn_varlen_func +from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import ( + Qwen2_5_VisionTransformerPretrainedModel +) +from .utils import default_weight_loader + + +def quick_gelu(x: torch.Tensor, inplace: bool = False) -> torch.Tensor: + return x * torch.sigmoid(1.702 * x) + + +class QuickGELU(nn.Module): + """Applies the Gaussian Error Linear Units function (w/ dummy inplace arg)""" + + def __init__(self, inplace: bool = False) -> None: + super(QuickGELU, self).__init__() + + def forward(self, input: torch.Tensor) -> torch.Tensor: + return quick_gelu(input) + + +# Copied from transformers.models.qwen2.modeling_qwen2.Qwen2RotaryEmbedding +class Qwen2RotaryEmbedding(nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float().to(device) / self.dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + # Build here to make `torch.jit.trace` work. + self._set_cos_sin_cache( + seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype() + ) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=device, dtype=torch.int64).type_as(self.inv_freq) + + freqs = torch.outer(t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + if seq_len > self.max_seq_len_cached: + self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype) + + return ( + self.cos_cached[:seq_len].to(dtype=x.dtype), + self.sin_cached[:seq_len].to(dtype=x.dtype), + ) + + +# Copied from transformers.models.llama.modeling_llama.rotate_half +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_multimodal_rotary_pos_emb(q, k, cos, sin, position_ids, mrope_section=None, unsqueeze_dim=1): + """Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`): + The position indices of the tokens corresponding to the query and key tensors. For example, this can be + used to pass offsetted position ids when working with a KV-cache. + mrope_section(`List(int)`): + Multimodal Sections for t,h,w in Multimodal inputs + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + if mrope_section: + cos = cos[position_ids] + sin = sin[position_ids] + mrope_section = mrope_section * 2 + cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze( + unsqueeze_dim + ) + sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze( + unsqueeze_dim + ) + else: + cos = cos[position_ids].unsqueeze(unsqueeze_dim) + sin = sin[position_ids].unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def apply_rotary_pos_emb_vision(tensor: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor: + cos = freqs.cos() + sin = freqs.sin() + # rotary_2 interleaved start + cos = cos.unsqueeze(1).repeat(1, 1, 2).unsqueeze(0) + sin = sin.unsqueeze(1).repeat(1, 1, 2).unsqueeze(0) + output = (tensor * cos) + (rotate_half(tensor) * sin) + # rotary_2 interleaved end + output = output.type_as(tensor) + return output + + +class VisionRotaryEmbedding(nn.Module): + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + self.dim = dim + self.theta = theta + inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self._seq_len_cached = 0 + self._freqs_cached = None + + def forward(self, seqlen: int) -> torch.Tensor: + seqlen *= 2 + self.inv_freq = 1.0 / ( + self.theta ** (torch.arange(0, self.dim, 2, dtype=torch.float, device=self.inv_freq.device) / self.dim) + ) + seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + freqs = seq.unsqueeze(1) * self.inv_freq.unsqueeze(0) + # freqs = torch.outer(seq, self.inv_freq) + return freqs[:seqlen] + + +class PatchEmbed(nn.Module): + def __init__( + self, + patch_size: int = 14, + temporal_patch_size: int = 2, + in_chans: int = 3, + hidden_size: int = 1152, + ) -> None: + super().__init__() + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.hidden_size = hidden_size + + kernel_size = [temporal_patch_size, patch_size, patch_size] + self.proj = nn.Conv3d(in_chans, hidden_size, kernel_size=kernel_size, stride=kernel_size, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + seqlen = x.shape[0] + x = x.view(seqlen, -1, self.temporal_patch_size, self.patch_size, self.patch_size) + x = self.proj(x).view(seqlen, self.hidden_size) + return x + + +class PatchMerger(nn.Module): + def __init__(self, dim: int, context_dim: int, spatial_merge_size: int = 2) -> None: + super().__init__() + self.out_hidden_size = context_dim * (spatial_merge_size**2) + self.ln_q = Qwen2RMSNorm(context_dim, eps=1e-6) + self.mlp = nn.Sequential( + nn.Linear(self.out_hidden_size, self.out_hidden_size), + nn.GELU(), + nn.Linear(self.out_hidden_size, dim), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.mlp(self.ln_q(x).view(-1, self.out_hidden_size)) + return x + + +class Qwen2RMSNorm(nn.Module): + def __init__(self, out_hidden_size, eps=1e-6): + """ + Qwen2RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(out_hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class VisionMlp(nn.Module): + def __init__(self, dim: int, hidden_dim: int): + super().__init__() + self.out_hidden_size = dim + self.intermediate_size = hidden_dim + self.gate_proj = nn.Linear(self.out_hidden_size, self.intermediate_size) + self.up_proj = nn.Linear(self.out_hidden_size, self.intermediate_size) + self.down_proj = nn.Linear(self.intermediate_size, self.out_hidden_size) + self.act_fn = QuickGELU() + + def forward(self, hidden_state): + return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) + + +class VisionAttention(nn.Module): + def __init__(self, dim: int, num_heads: int = 16, use_flash_attention: bool = False) -> None: + super().__init__() + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.use_flash_attention = use_flash_attention + self.qkv = nn.Linear(dim, dim * 3, bias=True) + self.proj = nn.Linear(dim, dim) + + def forward( + self, x: torch.Tensor, cu_seqlens: torch.Tensor, rotary_pos_emb: torch.Tensor = None, b=1 + ) -> torch.Tensor: + # return self.flash_forward(x, cu_seqlens, rotary_pos_emb) + + n = self.num_heads + d = self.head_dim + + N, _ = x.shape + + qkv = self.qkv(x) + qkv = qkv.reshape(N, 3, self.num_heads, -1) + q, k, v = qkv.split(1, dim=1) + + q = q.view(1, -1, n, d) + k = k.view(1, -1, n, d) + if rotary_pos_emb is not None: + q = apply_rotary_pos_emb_vision(q, rotary_pos_emb) + k = apply_rotary_pos_emb_vision(k, rotary_pos_emb) + q = q.view(b, -1, n, d) + k = k.view(b, -1, n, d) + v = v.view(b, -1, n, d) + + softmax_scale = math.pow(d, -0.25) + b = v.size(0) + q = q.view(b, -1, n, d) + k = k.view(b, -1, n, d) + v = v.view(b, -1, n, d) + + q = q.permute(0, 2, 1, 3) * softmax_scale + k = k.permute(0, 2, 3, 1) * softmax_scale + v = v.permute(0, 2, 1, 3) + + attn = torch.matmul(q, k) + attn = F.softmax(attn, dim=-1).type_as(attn) + x = torch.matmul(attn, v).permute(0, 2, 1, 3) + x = x.reshape(b, -1, n * d) + x = self.proj(x.contiguous()) + x = x.view(-1, n * d) + return x + + def flash_forward( + self, x: torch.Tensor, cu_seqlens: torch.Tensor, rotary_pos_emb: torch.Tensor = None + ) -> torch.Tensor: + L, _ = x.shape + q, k, v = self.qkv(x).reshape(L, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) + if rotary_pos_emb is not None: + q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0) + k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0) + + if flash_attn_varlen_func is not None and q.dtype in [torch.float16, torch.bfloat16]: + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item() + x = flash_attn_varlen_func(q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen) + x = x.reshape(L, -1) + else: + attention_mask = torch.zeros([1, L, L], device=q.device, dtype=torch.bool) + for i in range(1, len(cu_seqlens)): + attention_mask[..., cu_seqlens[i - 1] : cu_seqlens[i], cu_seqlens[i - 1] : cu_seqlens[i]] = True + q = q.transpose(0, 1) + k = k.transpose(0, 1) + v = v.transpose(0, 1) + x = F.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0).transpose(0, 1).reshape(L, -1) + x = self.proj(x) + return x + + +class Qwen2VLVisionBlock(nn.Module): + def __init__( + self, + dim: int, + num_heads: int, + intermediate_size: float, + norm_layer: nn.Module = partial(Qwen2RMSNorm, eps=1e-6), + use_flash_attention: bool = False, + ) -> None: + super().__init__() + self.norm1 = norm_layer(dim) + self.norm2 = norm_layer(dim) + + self.attn = VisionAttention(dim, num_heads=num_heads, use_flash_attention=use_flash_attention) + self.mlp = VisionMlp(dim=dim, hidden_dim=intermediate_size) + + def forward(self, x, cu_seqlens, rotary_pos_emb, b) -> torch.Tensor: + x = x + self.attn(self.norm1(x), cu_seqlens=cu_seqlens, rotary_pos_emb=rotary_pos_emb, b=b) + x = x + self.mlp(self.norm2(x)) + return x + +class Qwen2_5VisionTransformer(Qwen2_5_VisionTransformerPretrainedModel): + def __init__(self, config, *inputs, **kwargs): + super().__init__(config, *inputs, **kwargs) + + self.spatial_merge_size = config.spatial_merge_size + self.temporal_patch_size = config.temporal_patch_size + self.in_chans = config.in_chans + self.hidden_size = config.hidden_size + self.num_heads = config.num_heads + self.intermediate_size = config.intermediate_size + self.patch_size = config.patch_size + self.depth = config.depth + self.norm_layer = partial(Qwen2RMSNorm, eps=1e-6) + self.out_hidden_size = config.out_hidden_size + self.fullatt_block_indexes = config.fullatt_block_indexes + self.window_size = config.window_size + use_flash_attention = False + + self.spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size + + self.patch_embed = PatchEmbed( + patch_size=self.patch_size, + temporal_patch_size=self.temporal_patch_size, + in_chans=self.in_chans, + hidden_size=self.hidden_size, + ) + + head_dim = self.hidden_size // self.num_heads + self.rotary_pos_emb = VisionRotaryEmbedding(head_dim // 2) + + self.blocks = nn.ModuleList( + [ + Qwen2VLVisionBlock( + dim=self.hidden_size, + num_heads=self.num_heads, + intermediate_size=self.intermediate_size, + norm_layer=self.norm_layer, + use_flash_attention=use_flash_attention, + ) + for _ in range(self.depth) + ] + ) + self.merger = PatchMerger(dim=self.out_hidden_size, context_dim=self.hidden_size) + + def get_dtype(self) -> torch.dtype: + return self.blocks[0].mlp.fc2.weight.dtype + + def get_device(self) -> torch.device: + return self.blocks[0].mlp.fc2.weight.device + + def rot_pos_emb(self, grid_thw): + pos_ids = [] + for t, h, w in grid_thw: + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + hpos_ids = ( + hpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + .permute(0, 2, 1, 3) + .flatten() + ) + wpos_ids = ( + wpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + .permute(0, 2, 1, 3) + .flatten() + ) + pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) + pos_ids = torch.cat(pos_ids, dim=0) + max_grid_size = grid_thw[:, 1:].max() + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + return rotary_pos_emb + + def fix_attn_bias(self): + for blk in self.blocks: + blk.attn.qkv.bias = nn.Parameter( + blk.attn.qkv.bias.view(blk.attn.num_heads, 3, -1).transpose(0, 1).reshape(-1) + ) + + def get_window_index(self, grid_thw): + window_index: list = [] + cu_window_seqlens: list = [0] + window_index_id = 0 + vit_merger_window_size = self.window_size // self.spatial_merge_size // self.patch_size + + for grid_t, grid_h, grid_w in grid_thw: + llm_grid_h, llm_grid_w = grid_h // self.spatial_merge_size, grid_w // self.spatial_merge_size + index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(grid_t, llm_grid_h, llm_grid_w) + pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size + pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size + num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size + num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size + index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100) + index_padded = index_padded.reshape( + grid_t, num_windows_h, vit_merger_window_size, num_windows_w, vit_merger_window_size + ) + index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape( + grid_t, num_windows_h * num_windows_w, vit_merger_window_size, vit_merger_window_size + ) + seqlens = (index_padded != -100).sum([2, 3]).reshape(-1) + index_padded = index_padded.reshape(-1) + index_new = index_padded[index_padded != -100] + window_index.append(index_new + window_index_id) + cu_seqlens_tmp = seqlens.cumsum(0) * self.spatial_merge_unit + cu_window_seqlens[-1] + cu_window_seqlens.extend(cu_seqlens_tmp.tolist()) + window_index_id += (grid_t * llm_grid_h * llm_grid_w).item() + window_index = torch.cat(window_index, dim=0) + + return window_index, cu_window_seqlens + + def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, batch: torch.Tensor) -> torch.Tensor: + hidden_states = self.patch_embed(hidden_states) + rotary_pos_emb = self.rot_pos_emb(grid_thw) + window_index, cu_window_seqlens = self.get_window_index(grid_thw) + cu_window_seqlens = torch.tensor( + cu_window_seqlens, + device=hidden_states.device, + dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32, + ) + cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens) + + seq_len, _ = hidden_states.size() + hidden_states = hidden_states.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1) + hidden_states = hidden_states[window_index, :, :] + hidden_states = hidden_states.reshape(seq_len, -1) + rotary_pos_emb = rotary_pos_emb.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1) + rotary_pos_emb = rotary_pos_emb[window_index, :, :] + rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1) + + cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum( + dim=0, + # Select dtype based on the following factors: + # - FA2 requires that cu_seqlens_q must have dtype int32 + # - torch.onnx.export requires that cu_seqlens_q must have same dtype as grid_thw + # See https://github.com/huggingface/transformers/pull/34852 for more information + dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32, + ) + cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) + + for layer_num, blk in enumerate(self.blocks): + if layer_num in self.fullatt_block_indexes: + cu_seqlens_now = cu_seqlens + else: + cu_seqlens_now = cu_window_seqlens + + hidden_states = blk( + hidden_states, cu_seqlens=cu_seqlens_now, rotary_pos_emb=rotary_pos_emb, b=batch.size(0) + ) + hidden_states = self.merger(hidden_states) + reverse_indices = torch.argsort(window_index) + hidden_states = hidden_states[reverse_indices, :] + + return hidden_states + + def load_weights(self, weights): + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if not name.startswith("visual."): + continue + name = name.split("visual.")[1] + if "blocks" in name and "attn.proj.bias" in name: + continue + + # Note: only used for debug + if name not in params_dict.keys(): + continue + default_weight_loader(params_dict[name], loaded_weight) \ No newline at end of file diff --git a/multimodal/dashinfer_vlm/visual_embedding/utils.py b/multimodal/dashinfer_vlm/visual_embedding/utils.py new file mode 100644 index 000000000..f48234823 --- /dev/null +++ b/multimodal/dashinfer_vlm/visual_embedding/utils.py @@ -0,0 +1,21 @@ +import torch + +def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: + """Default weight loader.""" + try: + if param.numel() == 1 and loaded_weight.numel() == 1: + # Sometimes scalar values aren't considered tensors with shapes + # so if both param and loaded_weight are a scalar, + # "broadcast" instead of copy + param.data.fill_(loaded_weight.item()) + else: + assert param.size() == loaded_weight.size(), ( + f"Attempted to load weight ({loaded_weight.size()}) " + f"into parameter ({param.size()})" + ) + + param.data.copy_(loaded_weight) + except Exception: + # NOTE: This exception is added for the purpose of setting breakpoint to + # debug weight loading issues. + raise \ No newline at end of file diff --git a/multimodal/dashinfer_vlm/vl_inference/__init__.py b/multimodal/dashinfer_vlm/vl_inference/__init__.py index d7cabb50a..eaf8d1b54 100644 --- a/multimodal/dashinfer_vlm/vl_inference/__init__.py +++ b/multimodal/dashinfer_vlm/vl_inference/__init__.py @@ -2,4 +2,5 @@ Copyright (c) Alibaba, Inc. and its affiliates. @file __init__.py ''' -from ..visual_embedding.DFN_vit import Qwen2VisionTransformer \ No newline at end of file +from ..visual_embedding.DFN_vit import Qwen2VisionTransformer +from ..visual_embedding.DFN_vit_2_5 import Qwen2_5VisionTransformer \ No newline at end of file diff --git a/multimodal/dashinfer_vlm/vl_inference/utils/__init__.py b/multimodal/dashinfer_vlm/vl_inference/utils/__init__.py index 78565e442..8c268eed0 100644 --- a/multimodal/dashinfer_vlm/vl_inference/utils/__init__.py +++ b/multimodal/dashinfer_vlm/vl_inference/utils/__init__.py @@ -9,4 +9,4 @@ from .hie_allspark import * from .cache import * -from .. import Qwen2VisionTransformer \ No newline at end of file +from .. import Qwen2VisionTransformer, Qwen2_5VisionTransformer \ No newline at end of file diff --git a/multimodal/dashinfer_vlm/vl_inference/utils/model_loader.py b/multimodal/dashinfer_vlm/vl_inference/utils/model_loader.py index f6f6aaa41..bcf471eb8 100644 --- a/multimodal/dashinfer_vlm/vl_inference/utils/model_loader.py +++ b/multimodal/dashinfer_vlm/vl_inference/utils/model_loader.py @@ -160,7 +160,11 @@ def serialize( self.vision_model_path = os.path.join( model_output_dir, self.pretain_model_name + ".plan" ) - onnx_trt_obj = ONNX_TRT(self.hf_model_path) + if hasattr(self.hf_model_config, "architectures") and "Qwen2_5_VLForConditionalGeneration" in self.hf_model_config.architectures: + is_qwen_2_5= True + else: + is_qwen_2_5 = False + onnx_trt_obj = ONNX_TRT(self.hf_model_path, is_qwen_2_5=is_qwen_2_5) onnx_trt_obj.export_onnx(onnxFile) onnx_trt_obj.generate_trt_engine(onnxFile, self.vision_model_path) elif self.vision_engine == "transformers": diff --git a/multimodal/dashinfer_vlm/vl_inference/utils/trt/onnx_to_plan.py b/multimodal/dashinfer_vlm/vl_inference/utils/trt/onnx_to_plan.py index 1425d0f33..d23c53358 100644 --- a/multimodal/dashinfer_vlm/vl_inference/utils/trt/onnx_to_plan.py +++ b/multimodal/dashinfer_vlm/vl_inference/utils/trt/onnx_to_plan.py @@ -17,45 +17,52 @@ from typing import Any, Dict, List, Optional import contextlib from dataclasses import dataclass - +from transformers.models.qwen2_vl.configuration_qwen2_vl import Qwen2VLVisionConfig +from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import Qwen2_5_VLConfig import tensorrt as trt import torch -from .. import Qwen2VisionTransformer +from .. import Qwen2VisionTransformer, Qwen2_5VisionTransformer class ONNX_TRT: - def __init__(self, model_path=None): - from transformers.models.qwen2_vl.configuration_qwen2_vl import ( - Qwen2VLVisionConfig, - ) + def __init__(self, model_path=None, is_qwen_2_5=False): + self.is_qwen_2_5 = is_qwen_2_5 + if is_qwen_2_5: + self.config = Qwen2_5_VLConfig.from_pretrained( + model_path, trust_remote_code=True, revision=None, code_revision=None + ).vision_config + self.model_path = model_path + self.input_embed_dim = ( + self.config.in_channels + * self.config.temporal_patch_size + * self.config.patch_size + * self.config.patch_size + ) + else: + self.config = Qwen2VLVisionConfig.from_pretrained( + model_path, trust_remote_code=True, revision=None, code_revision=None + ) + self.model_path = model_path + self.input_embed_dim = ( + self.config.in_channels + * self.config.temporal_patch_size + * self.config.patch_size + * self.config.patch_size + ) - self.model_path = model_path - self.config = Qwen2VLVisionConfig.from_pretrained( - model_path, trust_remote_code=True, revision=None, code_revision=None - ) - self.input_embed_dim = ( - self.config.in_channels - * self.config.temporal_patch_size - * self.config.patch_size - * self.config.patch_size - ) def export_onnx(self, onnx_file_path): print("Start converting ONNX model!") - - # class SumModule(torch.nn.Module): - # def forward(self, x, y): - # x[0][0][0] = y[0][0][1] - # return torch.sum(x, dim=1) model_path = self.model_path config = self.config + vision_model = Qwen2_5VisionTransformer if self.is_qwen_2_5 else Qwen2VisionTransformer class WrapModel(torch.nn.Module): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self.vision_model = Qwen2VisionTransformer(config) + self.vision_model = vision_model(config) def get_weights_iterator(model): import glob diff --git a/multimodal/dashinfer_vlm/vl_inference/utils/trt/vit_process.py b/multimodal/dashinfer_vlm/vl_inference/utils/trt/vit_process.py index 651fe0ee0..f3ad39ce4 100644 --- a/multimodal/dashinfer_vlm/vl_inference/utils/trt/vit_process.py +++ b/multimodal/dashinfer_vlm/vl_inference/utils/trt/vit_process.py @@ -5,6 +5,7 @@ import tensorrt as trt import contextlib from dataclasses import dataclass +from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import Qwen2_5_VLVisionConfig logger = trt.Logger(trt.Logger.WARNING) @@ -300,12 +301,9 @@ def run( class VisualTRT_V2(HieModel_V2): def __init__(self, vit_engine_path, trt_vit_config, input=input): - print("loading qwen2-vit by pyhie") self.stream = torch.cuda.current_stream().cuda_stream - print(f"Loading engine from {vit_engine_path}") with open(vit_engine_path, "rb") as f: engine_buffer = f.read() - print(f"Creating session from engine {vit_engine_path}") self.session_vit = Session.from_serialized_engine(engine_buffer) self.device = torch.device("cuda") if torch.cuda.is_available() else "cpu" self.trt_vit_config = trt_vit_config @@ -320,34 +318,19 @@ def forward(self, images, grid_thw, batch, use_flashattn=True): "input": images.to(torch.float32), "grid_thw": grid_thw.to(torch.int64), } - # visual_output_info = self.session_vit.infer_shapes( - # [TensorInfo("input", trt.DataType.FLOAT, images.shape), TensorInfo("grid_thw", trt.DataType.INT64, grid_thw.shape)]) - # visual_outputs = { - # t.name: torch.empty(tuple(t.shape), - # dtype=trt_dtype_to_torch(t.dtype), - # device="cuda") - # for t in visual_output_info - # } self.session_vit.context.set_input_shape("input", images.shape) self.session_vit.context.set_input_shape("grid_thw", grid_thw.shape) - hidden_size = self.trt_vit_config.hidden_size - embed_dim = self.trt_vit_config.embed_dim + if isinstance(self.trt_vit_config, Qwen2_5_VLVisionConfig): + hidden_size = self.trt_vit_config.out_hidden_size + else: + hidden_size = self.trt_vit_config.hidden_size spatial_merge_size = self.trt_vit_config.spatial_merge_size - image_tokens = int( - visual_inputs["input"].shape[1] - * embed_dim - / (embed_dim * (spatial_merge_size**2)) - ) + image_tokens = int(visual_inputs["input"].shape[1] * (spatial_merge_size**2)) visual_outputs = { "output": torch.empty( (1, image_tokens, hidden_size), dtype=torch.float32, device="cuda" ) } - # profiler.start("ViT") ok = self.session_vit.run(visual_inputs, visual_outputs, self.stream) - # profiler.stop("ViT") - # Vit_time = profiler.elapsed_time_in_sec("ViT") - # print(f"TensorRT-LLM ViT latency: {Vit_time:3f} sec ") assert ok, "Runtime execution failed for vit session" - return visual_outputs["output"].squeeze(0).clone() diff --git a/multimodal/tests/benchmark_openai_api.py b/multimodal/tests/benchmark_openai_api.py index 1c6a3a587..b404725b0 100644 --- a/multimodal/tests/benchmark_openai_api.py +++ b/multimodal/tests/benchmark_openai_api.py @@ -44,9 +44,9 @@ class BenchRequest: class OpenAIAPIBenchmark: - def __init__(self, host, port) -> None: + def __init__(self) -> None: openai_api_key = "EMPTY" - openai_api_base = f"http://{host}:{port}/v1" + openai_api_base = "http://127.0.0.1:8000/v1" self.client = OpenAI( api_key=openai_api_key, @@ -269,8 +269,6 @@ def print_profiling_data(total_timecost): parser.add_argument("--image-nums-range", type=int, default=1) parser.add_argument("--frequency", type=float, default=1000) parser.add_argument("--batch-size", type=int, default=8) - parser.add_argument("--port", type=int, default=8000) - parser.add_argument("--host", type=str, default="localhost") args = parser.parse_args() ds = load_dataset("json", data_files=args.prompt_file, split="train") @@ -300,7 +298,7 @@ def print_profiling_data(total_timecost): image_list, qa, args.req_nums, args.multi_turn, response_lens, image_nums ) - model = OpenAIAPIBenchmark(args.host, args.port) + model = OpenAIAPIBenchmark() global_start = time.time() diff --git a/multimodal/tests/test.jpg b/multimodal/tests/test.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ce7c2a67c9731c79d2725b0fd3b9661f4ec8d61c GIT binary patch literal 50031 zcmb5Vc|26#A3r`b#xk;wQ5idfF`;Bg)-m?UFdLO5yO|NmlBEb)M`Yhh7-o#UEJabW zq=@%0V@ahbN|J1qwETP?zwhJm{rh_!=RD56_uTt>z3%IA&+9zT>-D<-X8z3s+-#$1 z0RR9V4?qC`01*H%jRpYk9Rc?me6Q^t<)s3I_j=X6rt`lGR%B~O$Gtu{jOKeGM9=p^ zunN-GS6js=MmOAB<*42fz`s>MGJ$#_;zBs}La2(t-W6l4BOd%ex&T0+0?-{GAg^Js zu*U-g04sq0PqzfHrA9=AhMOEd9B`iI8x;0G(ue&I2WlL}#$NKFUvvqyMxQ($8)F!0 z73Sn%Kl6Wyqz7{g%IZFWQ7ZhcGNJi`9A~!|K0(laCVM%0N@@6;Q!v}-z$JM zKmY{#U)?La@suNNRK0!je33INmqKxrUI8u)JjptL6^ zSO5t8pYs3z0s;U91i?bWA`sENaTlq*QxF&o761!_MfM~R0D=Uh0boI>s=kmF(OXzX z?IJ8OSLA5bU0HQ3>B;yvgwOmsWWNEK5|)&Az&f9K5bZm$py683s#)E(M^Ju`)&KbY zUxs_i+Ea><@ZKO=8UWfWz&(b7V35H71lp5RKozPFwjv73sCmPVUaS(rk`i+b?mqb@ z8|K5D|2GQ|+q)4c4Uz^}0)FJlt58;tHNoZU@)$EO{Fo@4bhM+Bs7e+cA_b)l@DeTf zi-;hM2{n-o$%O#?3D!^pIG{{~^dm#a!0*_gV@bQfC$gxg#l*;tA8p7J1%w*}-*fXPe4Q~h5?OUo!-~dfO4$iB4~fWYPz!We(F% zU#7e?{R?Z2`i**U*gr{HRz->?dXW%d4D)hO;P}*lgdXE^jsS>oqO*BDAYtN`s5J{l zIF*`fV_A(uR|D)Ga0TGwKyyy(I9Tx$5OiBJT{BZ>mkW|<+V8H@y}Te+&H+Md0Yu=~ zZM9^EQEkQyFA?_61+`CJuY5VJL;0f%TvVDX=Qok0nT=q0d~$wr&1$SBomG?!6e=M4 zn@o&wxj9i&IWXc0?J7X-SRw2ZTflj1KUm3qLXBZTNzca!qHU#}$iNMsb$6&Z zz?3Xw3n~f5fN2g{$m$iROOVm}>Hi&DjZL9oE^ZtBhm`H0d5_ z=|Wa3@wKJxj@1{6h*OyHKE7_q(i0s8Ou|UR(6= zdjeQ+|LK}sn3n+Nh$1z$;GO7246=eADsIdt!9XJ*nw|G=WJ1}MhqeH&FFjQIAHW20 z4Re&t(rLHGFWa>X_%lw^7H&c25=5@ykk1tAT8FeN0a@C1-6;wZ1p{$Z2e2sRG!2%E z1*k=!tIu-MYXXpf47R#^PIfyV>_~z#}RqP=h^T}Y4sTC zCVkj0*{%a1fG0__?6E?cNSBZZ8^U6h3b#zXo13edAk$6YL{wLP{7uD({-`t!BEdoG zvA3TCR?$M#XO$muD%3Tc>BLk~hftX&j5b?}T(lDu%@%6+!XJ^M_VRF2N2JuQi`OMC z+TCtPM0`S~nJ7O^7nL%DYQHa3tzrt4R!TX)8fSQKRf7^7t3k=4D&?O9rH83q6L68* z^hyPhTD6=B27(|av&{t2Y>i<8K$jxz-hABk#B@e9j~;iNRHZVgV#rKKkHY}vadkIv zM>TV0F!pe=f1OAb10b^OqRc11;9bBNZQ@}>)-gH&uM4R)CEEf-=ZZ+k^obH$2LM6C zV!l@kxSytqq~DGhx1(kA!2#5`dh4yK^h$LT9^80`Fi-|iOV0ZAtZJ5O1y6IA;m9H! zSF!auKv_9uRZ^biShZUcE9s~fJVC zlN@6tm-{48fAM4{Dq`g-4{|wpSv9JaoFiL}QSuoN5h^oTcD^cVPtURMfyj@S3yP*- z)J+L8r1MnDaej_n^DY5NNlmV8(q*Wrm+Qa971Y>w0O8wp+X!)5ko_iW^EYarP+3k8 z7EleD?yw@HK1~mD%GJCs3N?n0Go+gi!c;Jqs_|$V+7Tx3^d>K{1?Lr6Er0_q_wvc2 z=FVFJ3ACYDZ)d|rjB)MUEgCKXn=WF)yI@=E4y<<9?~r&k?vOZ~aB$jP1rdxh{0D$v z97`bLcX$jnFc$#k4O}Mof^sT_=L)5r7zY_ZW(t;i{WH>u3%JD2j<~qV{$6>IMPsLA~~=E(|9(mm!Y%_W@JI-kg;B)Fjb($T(im6h*Z)|qgIx19_Ikrgq^Fl5>O zrh+xEi`3-8$qXQw5n6Vo9Rx}jCBi@o>% zYrOX@Db(4N(?lr(i5s;a@T!m6-w{Mr%0j^mkBo_15_|zT(bikP6#t2}KxWI%ga6TnCH-2HSHuH7P9PoyUO)teDR8X-rnmw zNNLf1G+85KIe6gy)2kRV*@g@ULCc^hl2MfkLAo4Cv4~W4!E=0)r3+S^?5#*)Y&Slb z*t{j|XjXr^{F7@7PU6|SuqrjYLlp}GCrC>Vm(*HKuozHu9U@!XRw0U|_*q;RVyN<# z4iG>1mL=Bj+)hzTCx|lCV~H?qf{NNLx^^{Abr=bn)efPBF#&MkP>;pX4lmfIoFkk# zaVy9{oLaqtLED-IISU$B9*l==t#_EwU@Fw9+i2JhGPM8!4?9jybdl$Z!^sn%TMS@A zX)T;r6IMHd0djeXXbd?E2^ew6;-r_I#sf^hvzd!7pafALlRZ=0`n(b(JY14KmyM(X zR0sw9yDT6W571|!?f4XCLYB2Vv>b(JCo`n%NIq$L<#bY+57JRu2#FJ;)JR)*0Oh2y zWX&3eyC_IdvxLo3kA)(waPA;PZ3JsmR>HQo!rD(I+)!oZ@N-N^iCpeo zDe0*ihe{#pIJU|rj432h(7WO?Mit2Osc`q>eZI^slA4X-rSI^7&VnKm<)(xUW(Evs z!%&?l*;}fN_e?I(84VL)=)GP}C_`2rykINiAWrLL0K*t@b-lHTm?NT?f1CB!NbmSsh*zuE@$?K4A1=xvI&fNU59!$>xa9Jfg*A-&Ma^a z1W+#YaNIxfoT2f^ADz*&>(0^_uSLf&+EToWolV+3-)Km@0nN@1x3etL$DABhF(*6z zj#f~ev{rTJhoc_~Ke_7xo$+WMJ@1nR9!VECPNN-J0P2_6RVE?ah$C_jGCytou1E`9 z*B{cfsoO=bZ2yTLqNKBI$WCMyJ?0uIovEf>O$XLxNkb!8%31atA@gMhHJzPN2k@^N zG1U3rN3EtZs_$H=t;nHhCZsRphilLX$4U@zZNs1NRWc8fl>H zG}TJN-KYGs57OrvQ!Ov5N=*`1l3q!;9f3$^Cnu8X&q>{UIDMkf<)b$%Ve$C=t_kwPM!%beCoO=a3MpT?uG2F5@1AJ z+hAu@JM$4l6JB~*sAd8A4l>_H&`=7_6;-QXnUMzwqJp{sIkSa;_wLa(W$|9&W*p2L zj?zUjbVq0Vy;&fsh~*CvAUS?8ox1rn{z&M5-~)0GA03Eh*#85l`dMdR`|~HP`Y+Qw zE9U*+VR*}L`zLF60_EvLpojKF+xgZe753=Pd8kzu7Hjp2c*^fTvACYwLuubyv|g~8 z{BsT8clBOwzFM`9qTTpY_1mhu|Eum=Q;2>;VaLUg&u*_08lT`2lT4>)T-p}@DyK&) z_cHoJR@Zc2J_*(xDH>x9u%6E1+%kkTP+;k2)6YE3=aWyZH@$jsx_$D2=Zn-~fYmhh z)77lYXL>zN@6B64J6wd%RzupOch=^=EJo(6D|N3zz0ZK`NErBZ+Wd~hala1_EnWX= z{KSsFi(c>jp?rV#{+BXpc()*bx z>6iPu^?tpTx>urirgJ|uR*7bQ9sc9J@qZg_>#zUnU67kPHNAT^-^8dky~dDFUKwjY zZdJ<&KSARN_}AD<@ySYsDhKoNMM!3@EI?$6R#0kAcc$p0G1r-CTCqhXj3SXRE{s1Q zdcGZen#PFb=kjp=;-5s)OW0~z*5&967&}|Y-Wmdda}z{kgRmrDdM*)W_K=Wwv>YkX z$5KnqU;tDmuG!~zvGTjBe0ogBB?`1>YaA?5vIdZ)+7=>V7^pV>=>o^c^aApg;#?rIM~@S8d%{IbApKJ95`A`rP-c{@*X{|5*>CtG;_hm>ii*USig;K<#u)`ptM?{e#;wscTr zTG-GKo6uagT`jORC*!N2?O*G+QbQOy?6FIxt`!-d51lDqD8yithU zJ#V!*@-wTCAB8%%S*II(xUoR7bx<$(O~oT#jWWk@s{GroP%Cdv`Q_Uf3qBpf3;F3Md#tUFnLooOyEP@=$^Qv26F2rMXcT;q~x=W(AUM+JOZX zEx!lrp^|wud;0BH4wl>>?Yw=A8b7hLDDP3#qVfLgy)DfM(xYjKcaqq1U2V&G zp9*f|Ep;fkq~r}|tB0rgu-D!-O@`i^^VcNh0k;uZk;`s`=gRdDjt#C?956k#-%Rrqi^I&)X&r;3LYuoJE3DkUl#zg*@HgTLK2dSa$KzalAp*U3`u z{h=VQ$v@u@9{BK}(p!@PB(*;E$AF(_1@WHbj-mg{|Jf|!^L!ZHG9LM><-&s>$D${S zenr%xCwYOXx+zHXeOPq|MtTJ+C=D}IA;ZJi#EA-b-wpseyI>cuKV9ws=GHS1r1VNq z$%<`{H~>MoP4SDEE7^nnesb(|rU_a4tvf)yP>lGreSjdHJ5_Q`5jjk2E%}QUQD7I5 zZCi*ZJeesBF0yw8qlL>=OTju!H%UqN9FbMI5i6xy$DAv%<6p$u!XJhf;hQeW&`TBO{)& z9=eSrQcY57fJZPpL7vsSzQ-z7@$Yg*%FgSnKK}8l>AuiihxHw2aB{y0E^O=BT~FVC z0MgYT-0k1RLkF^_x6b_o(3B6)%-_Ed9VB$+m996k%OMoetG2po{`t<`$bcl7@QWci zOp6`at@~fJQ{CsHr55xK(zfDn4Y2eob*^si`(PV-;oc9i`Oc`!Cbho4*-=#A>J%|# z;*2AaoX2s1mm990JSj%JxvO;UsaZvTOw7%b+lIThv5`X|)+b`WE)IgQ`-x?05*6-a zeeP!(&1Re5oJiTuGwaK9+C%a$b-(d-nfK*YRPWxdVQ$GJD~LnlFLm z4=VZk26LU$H;lt{{`Sx05pt66U(?YPnl(N&UUo%!Ik0*jKk|HF_ABupAR^54f~d~V zL#yWBFAUCTJpAIgVi2+VfMGWOLBnz1^d#JcC{P&ZfGJ4AU-|n!BKhZ=?{48fo_D6_ z#vl9w-*|xf8TmZE;!CWBf2Z%&-M>5^^dm}jb9;8jM9eifT~*p4jt)SN+eFpSG5QvK zX68pcoB*=<2SE9hq3r1i;GiCejGZkSQwOlwuF}rl);Knnz|dBVsFS#6%>we=0Y~>R zPIbB5Y-)nIQI$H*{F5Y33dly=C2US4SJjcq$ZMo?w$Uqg<)8IC9Lkasvye}>Kv^b1 zn1krcT*Qj4DM8TAa5jNX<#hnw1=&fW0Elq}H-XbN2>6UYID73*iqu4kR-tiem1_8H z$~&ux)LR6wVKs;fT(pIA6P<_xYGuAeTLXf_JKi&`9@4`B@9wj|4IgS%bUGBo&>e88 z1gAP1AQa7C(I?BxckkhoTQNpo6jz(}`=2fNAfT8h+*KvEarl+edx?(q{yquSvoj+B zdlsaf<%m39Cc#-k07zsZry#ysB= zwlK^`80XmBF0A`m{Qj-n&!&Mi#iwgW3eKfUhg2l>Z>S=j76GzI7tV^u&iCSvEfY+{ z1}8HJjFUwj0f)q%+_w?-{GyBJ(AUcFcR`(;u0WtlUJgA0cX%N>K; z)z4Kmm%jTC=Ci-#KnCqZM)*I#dgjfsQNt`^R6S zcZJ_qNI!ZJ^^=^fz27(`tGrLi?j&^KU@*!V<5&(#SjrPAZK+C2Kk|KsG#!x)*2KBN zQsQ91mg2X87LfZ3%Fu4)0EIF+n62vb>#W0TRfAmXlA~T`uc!omk65+tJXS)}>>M_c$rE!%`37PYq_L=TJt_+2Cu2#X-3DTp35 zNo}80*_)7ax430M!T^y_25{|x=-b%rk_jL@WYI2`pv(doz|1~9M#-qheFB=4382Xc zZUYW%h&s=2qZA#HA&F+8z4)?X7bf<$25WX=1h5!!qe{3#YBm8;$`r=IvLi4cB*>8o zma$s3%f$-I6UFfWp>#aJn}XqAWI-cfK;8|O2ons4N?mioijl9)CWFNosvRN**p)h3 zYX?VFU`^W&u*1!(*z(a=u-@;~bU$xJqJ8sf94mE;u*kRKCRuSO-82@Up@9eQc6_tZKH7X&4?JcCgk}*EYt9n^Y(vcnF{fX1sHqEVi!j=Ppk+!)dZ*>g-Y~_gE1`%-sb`(WiPmWbryO`%Il=o zRUG08tjThM3_M#e49)325V5cD4*{0aN1=A_94m$k-)46EUUK;dXf)=kYm2o`Smcx+ zdOpgKV{aIZe}60S7qMX>_uhVRh@orS^Kkoz*`v2Jn@|we$xB7U0lsxE7hodvo8p&)Q3)LBg ze}E0%&NTMjg}cEw1l-q;UR0LSxuAFP`u>lCC)}pIfhp#B3dhUy1uZXu$F@QYBEuU- z&ZLDeAsaR;|G@8vwjv~V=;Kea^;Uiun^i14X6#6X7TM%~XPgT!UYGo3*7>sbbkCE! zFAm6^h{?olmbW9M;Yo#vJz?C1OuxIXp974pdq5)7jWRS|AIHB2neH#vTJ|iw*ZH`Q zFW%Juy&gUyQK!5IE>Pd62j&jl(bgXPs_gLRW&vybAE49ay<_yx3Vh4kKVyxSgMY6e z<{YrW-&ZoP&go1&8l6!(-an`IO_B9iE+(}*E!go`Q6`q;RI6H&r%rr*lI!8QF`pR~ zd}~057rNr7!ETzW70Y%Ua~V@PnO(ALby2*`sN4Bx*<6DzHd@79RHjEQpyUN1w-6gy zyXRRS6_KaQICs#jk0KpJCz4m2ObUmQvX~UyO+y_%y4{OWXu_w#XfOkW;+_xLiV+K< zlxgqqeR1g0AC;0<>AEQZraVDNM4sj>wq40|tODjb3qK<~-x+rVsY1Odf~4kJK1i(nkL)`2GpyjQ z31j;o;JRgim?h)H&&T_!x(yDVY`TO=U2T@NU|PPZM=URE<&-w7#?6Nu@ADUXAAM#@ zuO16)g{1*rvm6nMLme>_eY}_TWz`OCPYReNP|bS}^^`Z|z`zpP$x~M)@?^9buR_0o zjkpD$qIec#cldIF5vV>!rufLU4+z@Fj`WZd+M))=5ck6M5HwqS4 zFI~2kF9C{Nf1DCg6Z#ME!+s*=^*)_XkQ?;d;5`T7Pw77|8#UhfQDuZ4ls^@|`46CT z`#rrc3YeAulVTRc)xLH2w^;rUL}b&|{rC6RZ%n9q?4I=Pd2)hEYE8m4U(I}X;^HI@ zG1hVYE~)*jOQ2y_u=@+SxtM9zqZf}1Jv(BDBz&567nbarJUmAnc&Hm|61XL_S^ZVX z_~+T{g5}Lx0|%47y|T|!z?VM^53MQBKy5MY9?G6yF}D4WzY6;Z7JdC~8BS48I6#k^|z%w8OdPMaut+9M-9#^>{EW2MXu)oASG%x`SX zf{8^e(q>TDBxVBXIG3F5@`A9$W9iT4>SMI0N=jjTjk8G1=4?W}tr`YqLdZ_8#_VBs zl)Xz61dZ_VWr4EPV!4IR{C#)YE|W%AiT}~bv#F^^>6ogHJzwmEPYT2;-_{ULeZnOO zNwmkL*oaFRU>wAy(13A)MuL!YyA_!jLhT|5$$ruPkV7|4WI+Q`1eoQ>$XYzW%5#oC zPwA(VF;ZqrxvpsPVwI)$M|TWY+JPbEXfkzkfh{$s-}kNN`j(gSVdKxXDd(zmUS6VdmDK9=5^~QUk}EwpEx^y zFKu5RdU+k^nsQu-W7TBPC37}A-SFXc zZ3#g{f{oh&$T#^@_vO65Ji7ipBYZSe$=ian)2nRqs`$K!$>qs3!Akvs>zs5Vc3l6} zZxpp4;x$jJ+-#W`z!d#8IV$DF6JMsrkR2F+x{T>qkgX9NsxUE1Tw>teH^qqZYM{apER!(yYqLBG@KnQ1vF!b~vdF z%BlS1f3T_)eXr8wG9jhv!#{vsk?-_+RW6^D%O~`d?D?Uz6U;uPp`;Y0WUM;&VmN?2 z6<%%p8|}IY2n?8|{d7xzE?pGR4!WhpS4*c2r(lKBF=(OmU@S5x=x!~XAi$E+$DYFG zuwm|a5rr^j70miq8L*Ovq}k))c6@N)EXVjR=GD=1!_y|aZUa_~sZNeyXCLGxVqm$) z`V)M@Zd`xH%^JV>L3^Uy$3V1Hr6yvP!t5!tzrLpVw*UG?+tb2*4-7O;JN$9)njWiJ z;21f)VAX9&-FSp!q>rC@5w*gK6WYb0Erk@K8{A_Cm zEonzJ#;v4CzCZo^jK<}P(~m~+4;~Bt40c|U&@F73GE7>iT$z1vqLH&m4BZ+5ng5#B z+Eq2K{X5ou@bjB9pU<{2^h_FuEt|u`HG^7=^M7FAZW%=K0nM9|F;!%LO)<)1=jwkf zoqTv{gMj>VsX(1ALnBb$nCW?D#DB%kshO%Kxt1umJ8xE;vqbkpluZPxHfoTUxJp-X zv3DymqxzO6!P;4ZH%jpOY*V;my=^vLhmZI}OXf@Mr?N+zw+yrvov6a7zbPtBJ|&w~ zl_CvfC)X(&$GI;dERv0aX~iwKU8+>XEju~48#t5-{G=qdMOK;cMuB{81>a_wSQu<` z6}5+R&t~egN_cVFaj1PvL_?L538%(bk@tM(+2(!J%oF=;+}-aVhhwp%(xYBL0aB1EPaFL7cZw3x z0F*e8lz$ZUoCuE3K~yx^Gy#u#g~U=))PK~zCa^S6ydBpB1X3Cx_i=(5y6ExcGW?nL zb{tPJ%jZnhR=b6&<3+C12Pc||&38sZgpq7{&7pd$I?cOB26sy09z5=}L=B^OJszFw z4&47@ZO?Lq)!l=a1D37P0s~|zZFWgBQkdamxHq#=l?t0xY+?xu!2rT-czE4GkeA|e z*SG}F+Jv9r%r|h3LYs>2J6Hzwn$UD2D{>AFtYGo_23Qv57Z}DMAq{pX{#$cJxB9Kx zlz+0X&#>UD?8V#VNpRlA3P#n|$_5D%O22@$6h*IM%<#t2E{}DQ`E^xpI<3eAH1^g7 zti`)U<}huneM^_yOI~D9IGCiJ^$k_%pIk`}k;0=XOIrFEcQTAU;tA&KM(fC4{&o+s78CAqyb%L#E9wq+g zcTg*#18QzEp=#$9jPZr_*_~8B_onRWW7A!e53KNAr)8=%k4-(^I+*!1ViwR zo1PJjQ6Xj-m7><9&3N#M3dwg0NVztO&X_6@#odUN_PHIwyH~)^ezii9+JujySDc)( z7SfJ@62f~RY9$TsP@m*%CX!O`6PIO2v2_$vqwYZz+S_u7356&xt(IJ$T)#J(_w34> z6rDFI5+VFHi2}WV8Ys#piwccgBMHcogyDNpcRKbK9I6lJWr>@uxxhf649%Oxu@Mu` zL@wA_irz4kLR>YG=r3UkyJsGY)#0N~pc5ym_tgqrCgfpgOUQuYvE-DpLP52r%LGn1 zsHF5eyUc?lGUw*zVGg~DnpCd{ZU`KZ%^TeOGdl&&Sa$W(PtfK=EcnZA3l{Bdx2>Jo zd4<~Cq`1$brM@HH-CrLR4XJin^i>7!zZ)5JzOh#CGeAnxdZK&hd&;`}!EcXK@0x}E zS{=C}o>FW1X{qfsN0)}#*2bEjke6n~5L@ou5b_=&e~&gYxmgf9I$|s0o<0>gb~K`K z^_SXb%kbsa8M*L#?VU+zg36RS=j7vw_QT?DpRVuQ7qt88EyHIqb?$s!cS8yS=|qiC zFWT+Z z_#t&D_$OqBUT2J#aIQ2M)r`2(;Mse2By2$|;=lJNS?;bGHhB=XW->gC>Vlm}W-}tH z_A;xqHe4B@&qmv1w+r{+6C7{zEwiMBV`X}9`yq45`RG{1 zPY%CGz2If5HM`~ld7_kv2^YCSQl!ER>kwHsk(DTmExIh6nOxPTw6#~qXGqz7%u%s# z^(yrlD5g+E{V5I6YEkuatVDsgJ0;1Z_cJo7LssgD_GSd<4qkE_>`1Zq1neO#ZcOK z@GhwwX>Dhu(|PyQ3WyUnR=aZY;BaDxKz(YVv#dh$w#npaqK0}=&vIT(vWUT49-~0G zrc9Ql9@^rCYRn%~zA!?0{5^UBa5gwAI3v>gwEg#@`w7mECO6XOzgJ0q96aq(buR49 zWc+`>nUC)L={bGkZQ%8oy!SeVH&31U>W9f>XV)&CLrx|eE_F(1dF7B=2$6o`k~Xfz zHqAG>;*}N^B>U^FP*(lHSO@vPEZNv!pkW75$IlYX`e@mLGv8b!tMbCsB|306VPb65 zd|z1Z{V^t^((#4Ry0cp;lqzE0v51}^p=G{MA^Q>rq=gbcTq zR81O11dmH?8mSq>~uuo{js;ycT zj`L*I;B+2NY?qitM{t5|^60$P{e+$4V?iX5mba=%7+)qC$Q0uQKWufJm zMf80~F8YIthQjFG;H;P_H=VkRcZbEj5zl5(Ev+EZaI&JXFO1ln@wO1?mfNFr!MRSW z$y#Op>iG+j>;FN!4?O+mEPuy1`%vgXvfDEP?$)zk`zsNN$4Oq{CMU;MEbsi6Ra{3r zDe(GL02n{H*oJZVx*C^)?+wP02%=0wr@?~*3IbmfOjSsKGCtMK2iZ3o2a0q=TfaDc zXC%*Ro7K)bmu~2_Ea!j%e@BNKn3R7yS%7$^jN_;~m|t4K=PVtuU$n_NjwCdpUZ|x1 zs8f+DWMMHDPiwXkqR=W0SeupMyyTK33s;qZf-^B40xHPMjF;h~^tsb;HvT-mfu&zpca6Xn|LXfqEc!*O4h*MOpSZL>WHtH847b^6M zo3EB!2CdJlE+lAs3?41SI5X*S`}F!*#OEs~^jBOBU!q*4e^dLieA0w}j#niX$ryhscDKQM0GNxS1Mb5(PUCL>U1&t|A&`p^==n zy|9Ie6-A|a&|$N8PGv;+i=N4?x!_z$$1dX zrt^FY*{VM?O;N%OWX_|0W0ACeWnTq*nEx^%4c@Qbxs9ms?kK9KAJFlXRZvM8<>hguC?Oy5Dn! z9?Dt3YDo;An|eyvx7?vPpdn=_R^{(3H~CcB!wHGyQ^4hqHcD#*x!Vf)HVV?GfS;I> zqIv8Kr7WqlJ~0!=n3CK~1JK`y(<_o=i8jSG7cl;;*wxKi+|u1Ev!P!f#38MUWScCa z8@-lpj>n`T9WCfr2HcFpJ-4jls8-h^DEEuIT~Hainp#Le5J{KRAp^f1P@?26@XXDn zmEhxL1*ryP;W`-E7!|7@vMMsWdRGQd2&BZ?`9HlG#s+Gfi(VP44Vi%joEpyUA~G~z zd2fn!rb%L&;tK(TCLz^|DR!L>Dp0@6ZYMZM1T1mwRD;7bS`+}X%5c8Dg;NAQvMbO$ z4i8Hn7~*mh_UAsxtKKNXb1JpeTK@rfEGY+gLQ6CMoi>8bStcc_sV*)-aIQrVhu7P6 zC0l}!k3F<2K?A2A0VBP-4cX0do)O)z>QqXu*U0^--?fC@_9k9UHF<8@Xr-+C*d&`q z%Pah{e4JfUf9O>IZ>uL4c&ype6vq>fN_NUE;XOXvH`#gb^)#OHHBgQ@_P*}y3J%*Z zK;=8@b^^y%gaW^V5S_|(D^%EISG3@y1wq*+(sq?|k^ zZoc4%j_P}qbMW~w1%sm~P#bzMmH^$!Uv=`}#SW2kJ0x$aWkFq*R(c>&JyuJ2&@F{J zOrbMDd*%lRwrEouL*bq+^}G=4c_IAN?ZA%12D1&-=f0j~FBq9nGCKK6k))e9%Hikg z97d16_OQ!C4n7~LtFQc72mJ?#CF&O4j>G9$xRQ#u_i{!pS@?ar(C-CSpBqp7ieyUu z;4qttXjcp2p1jyb$&G4BS`RYmnfbkT+Lc__s`aU?!;vh|GfNiSv;M|M_BdQ+JZ1D(qly%d(bKw zdev`nQFxPip$qW6@$x4Lj}tKlQi!3klvl02?#Z{9zV4Q~`~Ku2!sRFSFGEZuz47^h zD(i=br2A-p`KS7Hhkw_+_0@VxMJ8Zo%XUP?At);MXI+T4)Sb^vn60pFkAb-Bg&OELJZZ<5HiLQ&r9Gg?+yi!UtY(i zuUOIN&P4bN-^(QxaDbipQ;pYmVfS(Yxf@j^u2X=Ft%=oK5iR%bSER_tqwyP7ip#w! z)Job$MjlY0rCwCY_2d#rJTQ^vz79iLX=^CWs@+_db zbZ644`AukW`KLmo*W0&hztu;6JNm^#@6?;F^LOuBvn@^LxU&?=uDMbt#BEoFlYXzz z2SPgUO9BjHZ{~>on#gnURsT69q5eMqHM-wnFf{Gj*MrvS7KL`dl%GL^YMLiImsi~n zE4zt0w@qoWY-pl9cG9!) zbbtySV(vGtfq|q~nuvg0+*R&^oTiV*KgYJosPO8wysY+8;1|JAb(T^-n0lG3Wc?4| z@ZR}#^vN_M-oEGtnfqJl6kQwt=r0zon;)+kUzR#4;~iutex>pL`NldU=Yzn3y!h3) zeS1%(EDjc@+`BRR+LZooAegvxbffHu^#($^DzVo4oLTf`+3qNK8}9C*G|^^v-U8q8 zkI&VflbJPm<Ok0|25U?<*7U0MYbnD&Lmwgny&N!A5eKt!Q zd*Zn@z-|#Iyy4g)de$ez!{|N;`V5El<2)q*;Mb-C7d@j2i$e1V5mC`Us|~}<`yQDM z=$$zfu--1dkxGhKnYtr0(AmG^VO;+jyDbO8+$cDCeJ?-d^##EJ$pN1qs%eShuTq9# z^H9;G#0(HR+jDzd=l50TgG(o0y*PP1r{9L7gEA@yN0cy@VwMW;Md%P8RFBs`)juE) zxwJl*RrBD|N)zMchdEG29mR*8~=UEBt_GT#76)x1rYl{oz z6b1n5&PZ>}Hf$c|w=%w)Ea=NW?FDYvSyz+t0X5YQcm4^cYqK zj_W7A<}to<`Ao7nHWqgKYG3P-p!=bw9oikm!ERSiO*p@uQyhAK_SA3o8<%RxPkyu5 zr^V3YgksYv`6y}}kV+sqaT5eYxODSqob$&8eO`g1dmcN3jO%B2~z09dyV3Ictuh&L$`0G{s{d8wIl&`h! z8P31zHI6sFka5$v_)j20g z@kS;h#=-fBA3JWpGc!P9(-ZR}V(TA;V&wEkVD%22_-6z~y3K1^4>@i7?OY9hDEthu zxGqG`l%Mo2tiIrNSe;C`!YA9J<<2!0?q;qlq>~q0cz9cF&m6rxrm&QC47_PbxtV3_ z{ocbduzx1#{&e$>)3Gj)qgjH&^JT(vaOVB$fCGDR)sMdnKb}7`q;Ph7;n>0Gdu?rg z$w$*l^~@Z02hX*dURC`3U#i8p(X{iOr)ou_QzX$D4N<`6v%ug$;z(0yi-Amq_&>lk z*1HNN`xuEIzxatevbL$dQ>UuEFBPAEeZBH;lUK>tW6CCXPUN>(OiqW_Togh4f6v`ayY+>Ap)Lsg;;HPXFCn^Uo`3aTknZlC$V}%Xygt2esDp zVg}_L34EYau9PU_A^eEpE8LPHM&$-t0FkS1JgiKqIpQ^34?c?hT;crhPdb>g;;ouTSK= zDirm`37c9o2tS2bCg8J^#VW;m2jzAjr2IkIyn-6$&_-L69N?5fv(@iU9-T20PSp>o zELQ3!Rc4Il%3l&j%S;3fn4KID>Hk*aJ+-Fs#dBU&{EX`9$#Q#bVwS^;i1LL)lm7tc zA06@)TJV(V`zk?LY+GZy0=4s(be_*>JRUfC>&`1vJ;Wnld7qaZt?f0f3tSA+v=xkGZ$)J9Cj?0bC^=ko*e6(twHna`8b&V}FTD1Tk1ipM0ImV- zcDJ)9UxY^8g^oTf`r(Mw#^nYZ+@fkmH7w`}ekR@fXhNaX9hE3^;*)O#yuHZ|Hk#>g zdZQcCv&aOqoc$-FMgZZ9Q}_2tAnv>7WwsRD15WlyI+ragqdv|U?O*+~a1&>=EqK^_ zZL{gE=(nY5xXSFDM&=!Jh`WH*@}Hb)rLOYheLmgq4x4dWZLg~J9qM;LPFdM3l8BsX2u@4DQV5eYx0UE&vf&GNC_4`jt=UL=cC@z z*$Vpx#p<>=XBHgXBR1yv0N<}UrsrcPJ${^(Rys_3dGp%Onm=F7M$3Di_;8Kt?!kZE zcw-<_V-wZHSQ}{A4Rjw~zM%j72d?Jte;pU^zr;5qM^ZE=b_)B`NjNM;q&#NowdcWy zQBI#I6XDPFOAby4t4xjO7h?&@QC~xh;*ZX0{=Bz6w6cLHP}%()&^-BGNyD%Iq-Yn$ z!LkuzbNf$F;%j5+bd%3TTn!ZsTYGld?j~pTV)1L*(pjAA@~~`OzNzx!ENjD#W?RiG z5uVMuuKB?r9vI{=B^p`o6f|8JH7E8yNaA>}?)nu&|4UUp4YW=hYPd1N9qLl zEk|NSz=_Y|Wxuwx#*K{qG)R48mX{cLF?0i3vzQz>jQ(NuW3Hz~81dn?{P5PG7E&bad{3u+t@Uye_=d44PisC#$pm+OA@r!y9Ep$EGL;MNcQ zpqw8Wj1R{D0NGR1t7p~Tye_&Om{%ou$VD5m{mEq7wDY~G)1}p$XN-!MS}4L4DbwC* z=cVpjb5V*uZew5KO|O2YZv82SznjT#@W1Z-a)|uEtXHBZ>vXZz$DWwzm)$Z-R%aZp zkBe$sc8dU>K2AA*<@iu%u|(57Z27}`OZ~|?xszIj{+SDB8gu`C>h8vSDDRlgmQN&f zvDqDGBrN*%_t}`FQ3`Ya4mxhv?1ulS?R_`3LX2#N)z|2U~|vsdw1eNx1{R)X#9;f z<-?-44|7-QF4w8CPP8>n+V;%E)#z$XREm6WEO<>j7jdus%23nM!NYcKEL>Wx5B>iG zd_aT0xi3r(RN?J3J94EJdMyLJ>O{S*RtK~ZQYH9K;-Z0@cZsuy+PQH1fu>tdWhn9$ z$2ixDSH#z8LeLXz$pyQ01DPg9r1d|B)tAToP;ncz*fP1dVsiniR-06^c|J!ee7JOvPX;9zsmO`T6tt_QzND$5(ovB{QdCtTN4SXj z#b}qfYqytS#i7M1CJAn1FYJIPwz}6*O`4aTNx4vPNmL)z5fBp`qxyqd_0`kD<)e|C zkP1|`5U`Yu43RS%1HUTr9JtB9Q@JaM+%Jq`_!8NA-sNmwFmneKsR#S21Z^KJ>(E{s zc)xGO*2SCDqQ?_z1C1qKl@3b)>H+WduQguc%EMl9)-cPfvZ5L$0uX{kr&SUT>KjKt zY(5qJHE)lMM6>YM4Gii)bS#vZ-Xed5`&ULEB6!}Oj(nJ-I??WCZ&2cMAz?}C4AjYf zM?+j*!;O9d-M>FODsTk{P-ZlpPxUprU_5JUind|Q#XGJbAD1fiae0u*^vg#i{CP!M z*}Dhx5nI0nV3&{EIQPRWA!KDzl0YIi9jjR2i)&l#K)rls(SR0FG3G0heY7P10F7uq zI&sa-*BLU+vYoNA!_t)6rG=@m3V2Ci354U491g(YV z0QH3t{A5-$lw%2C48RYWHOGWhvYPeC|ML=u` zCY(5;HTnTjWc`md{kZr+yjR3t6UGyj>^WFH&K&N5q%6LLHRfDsp>_ocbtx(myy+zX5hTwk zit@acl$$%Sbdk+**41}*Rvtr@!UOJyE;$L5! ztxRP|FqG;>pifg+WXlHGv|EHsz7*lL;W$%kakoO;^{!DMk~JhBpdUD^>nl{cfP@KI zCJ^B%R**DTQ|~o7xpLfO869;>(JE0IYbHkeAH+~J>TYex1uB3_)hBb){CU?iojW<( za_qit>ccK3pcHBkQO<>>$a4F`EIQ0Ssax~9G8eJtbQu; z=?e{nDLFt;R*}A0)17Mf7slSz*> zDMI8(LS_#=X1oW7GMoH)0AxbN!V#2%DwP0C^WJOII94?6!opm2PC+?T3;;Py{HoB; zJRgu6!|WyzScR)h(3sL^O?p=CV)c;tWrholrLRrPWhpBFLO})(&w7kYO*1^YX`^at)eT!E zR`+Yn1Q3!8sD51k0F_N@^A{mj@1erA0(`@|M$;r|wyerXsl(cEtQA9(o5FM7dd@F4k)Djh}3>H z&+zXRacmcg@eCspWiGzdI@+Af1Feyw{X~h}7(1la#CWgteZqtlw>d~kPP$Hn?lt9D zc)J)M6|mYamM!lNVQ%~q;tx1IHk9`H)?8e4Z3#uYeH7eX;~N+i<+Dg-wB1}J6f^1s zsH-pUM9fF{*6W3)+VPa8{kB&6(DO>bAr~16YbQY&gG**(rWG>kXg$Qa+}zx9oQDUgpm5RoL>>phN%&f)ZzL{{8Awu)=cr zGn8g9BuDR5Xh}j6>KiCP`=CgV>!kreX|)nO)n$;G0F$WRv67{unk>k1rijkm29%u8 zKrzqgYNF(+4k!l{fI!X?eKH1ze)Ct>>Uq`xS_p4YGy-{$ed?1Ew@VHBP*9R`W#|Vj zwdwP$B^y?aV*L%9hH;?ke3x9z;ngIa>tqgPGhELK#1Ua|{nD00X#lMN?W}FSonp82 zz9Ul*Vk_-_8Jbye)V&X!^RdPmZa$@uc1HTqq0M)!7Ttu&HD7zgjoMr(gf_PdQ<-ZT zB^@O7_ODMSNMlzgM+%Il=^GGtoYbCdC;3*Wo+nzAge55jVnGr@dl{@t-x6@F3oBTq z+igBXkkBdsPVu-KZ(PgE!zW3ae2K2nuHC)y0n3Fp26L(igr!Ga`fI;EYnyRJA4M$$~fg#HJ+dYsfr9 z#IEBwg4W~6Qf3gANb-h)U~>HH!p9qKO!cSf6nb}tKG57Kls$*ykKEZMD$1MZk>+{5 z%2GWu$^=&LgYmZp@uS6@9~ZQ{3jnRKVZe|z8C3_DO{>ft6X4DN021(x?-Sms%RD8d z<+o64I<8f)jetNuw0G7!^E;cLBIJhiGV5f3qH?Gn+{I%o3T?QvMLsH-^cMRD!Wpqj zT~ZRrMxdxEQPf8%gW9ri{5HnT)>h-~JfM=PNPwi1J5KS=Yr|Ex;+`C^)uxV^%Nt&& ze#fzwk!)GBTKt9RrG3$?Na{KA+-+FgCy76XT_C0A!V<4SvuVD@H~RDTp~ARR7`kDf zKHJTGO+%js=4OFB(zO5&%y!ncn{N)SlWe78SDyfw%tK2BLO;A<4Y>;BT-&NP;~#T# zv+;fG1ZL4`BUaNnRzT)7k8xC{al#Ex(izp7p?@uV8oQ%DJ}^_{nhEa$7ppvR1Dwq$H&&`6d(3c|SVs zhFpzsdq*s>K|XyL95;^RxcY_9=qz5@ZmU-Btd(t$8Q;vy&}b$rZgYk4H(pD(OS88q zG8DVLPi&%K{dtPjcx~|OX}wkLwb^zvC}jywonz`xeJbY{;s4T6eV}UZ5ouaNTVb`jK_(0+L}}i!?J=ucOh`RviQ*R5EX*t? zByFy3PxbPxM$e3KAJVF}xLTIdpft1qlA*q%U#{mswaZiyM_RjEcBazFkIm8Vdji(J|fBi;t6!QC_ zU3Ap)u0r-6m{i>;f@LMb6p`*my!Wa*`@3dBoNmom7X)PTl&r0Jn@Q?9{VRW&l_Zpm zeqJXoj?}-j_Y>h^_kgfd&V;<;UP6?R07?N$3!a4N35W!ebtIaX3;TFmVQXccBElMQ zxU8Yve92oKiwUk|WYif=>OQX1WC%GYV3FYg3?B znoJt%4f&0h?HeU6G9e8%f|R6AkT%{m)17h_xT)6MQ@&!96)Gw!B&g2hb=2wg6}<4a zAiK*(%Ss0#K`CCjPMdqd)0KI-Q>sT?UG1}SZr0!^B~2l;ec}oA2;V`c`PGA#>RWA< zdXW%R*_ij%tF5jsmXtREMH{yepm`N^-Q~ra zazl$kM38cY846C2W<9s8%T=XBDTU@yIfM^=&cpnTVJ(HY;aZ+ZWh#hCFg4eItw@U& zi;e{>xB|f-kU@@}KV-y{%Tddm zbJDCIGn5pPL_y6EM_y-7tzrijwW&d*wnE4%h$er_YLvaowmqfY)rnW?lW@Y53r@LK zKXymYV>La!wU!{$s>762;Zn2MTs!*2^-53e{subkL8nRwujX@Cv=>&aiv0(f_i?xQpNh(lC z2T0Rc6|B{o!O|{n@w6fPOU6`C+QOMur6pX%_0=cqTK@nWVvb@MQ3!B2+L9G19JkQ? zsyu1;8^V|5tr?nyAgFp!W^~qLm(IDqE#f9Qbn4l{aMY!*x?Q~0GqHdUqrB_VrYq~w z5_b}Vhd8Nta<1$hC6qI89mu7TGq#-d2kTu^g}g<#zqfIOtyaoP0W+vqZA|I?ZC-cd zXKwI&^x`15ad4mmiHxaSm>_=*>uK=z$+ceM)t_nNWiWT5@MHWDQC5>(A@Ga=Z(Pnqf$6EFr<7 z+;{~L2m*45(^E6cYTQ0I6I;!k_Q8i7pDiO&MCAcn{y$oWQQI+V*tcg7ZN-M41xZ-T zI>7e_wO6xYePM}tt9?HL0UkmTDhiM^B$leWU5pvvIAKQng6q9zNjTqj--G#u{x3H%yCX1FH0uAeACd z{Hi*3t#gNSWLu4;wvMQ@~=5r{h|K=N#cthat3vh1M2}O8t5aGb(#_4+jlo#80I$9ZlOUU-6B0R>PR`a z&_oZyvv?c)$2!UB6Vg|TeF;i)B;}mB;G=P={ARK4@5|v!eQ6;n2Qtl)U8HF_Uk?NI|`t0#MU^a{6Jy1f1|ElCDmIzl@?T@XVn`Y`Fi~69~w;u znIpb&UNLWNf?Yb(X}EpX#k4Z$QnIN*KJlnJbT!9(Me&D=_*MKfhV$H5gP4AiWQD5b z2|u#9-&iAk2Gv2sxSk`3afOAhCequuZNlOlVOfx%l*#o21q2-jBhtFP_8zP7maZF3 z?TXdrEjN%{PTGPrnA<>f(rb84_}@kLWa6^@RCAl39{7^V-K95eiCT{`1Ma$qSvtfL z422%2LTjLKSBlqo?l#sDag#AenBvtVCX!@EvPjU?e-U77v}I|tVZ-L4QdJ5G9Sp?l z2DI-C_(vJQIFAy$aaM@7y=Z1CCC3A^$Z}3nPVfouRhCHB-|iMUOQd=p=JwULPn1o? z#7L3pN4K}*Tw9BHZM;=)&NAaJsocvBv}IB`kgt7XZ7Z(f_RKn&2~)X;q!omZqG0Q- zPM!h6@mDk5;jS$Qa?{I9#?og>=ln!q=^Ff*PiK=ad=uk7A;i^e&CQx@#sI9<1V=>xeQyX`$MdF?vaJ9Hm__r|S z5K)nf#X3xKg$Rv(t5D*<4TZBotS=II#4FO2-y|h@v>=5hLO?KNk8>5%9P*x)jAafx zE_kzxu&e$bZv?V!IBl9*47L!|B_k{V03ra=1#+xA3d8V>O5w#L54E;n=kr^)NiJ(5 z1Wc2;CM5H%dy8><%p9q^ww!L^AO*DQV1-2g0Fmhx&$wHNoWyaPw(f2na@{XyZK)+B z{0%aXu~b3RQ#QEgV(UDJ@*3lo#eU3IJu)c8Z_FehC;xEo+YP zP1~5RGv<6lRn|Vu({sT-#g)DP?79X+zv?s1?T-O04N`rAJ}6;^gE;XB$JR5|1@e1KZxUZY1KR zT3bDI`C6SJMJ=sfnGq2>B>VNOOeKbmwZr(2p4G5P0mjf$w2}Ks)1=m~#GE^KiM{=v z=wDiv^Avxxd`uZBCKVzM!<9mg`zMp^)p6cBy~aau!0Cs1lOea-;%~4p}3jfz!FEu>4Zv z(x;M?t>UNx0049+)j!s?+*)b2Jiyz3sQ`Yh3G{*9K3^)-_=&6?>AJdUyKo1owuCsN zzos5y$bHqVgGIR1%e?K}ZM@8;}8> zk^JjuyImA+pJw4|^@=T&w{93}7x&9~TuAPD$=D9HCMEhe1+@=YGivLXDa`Bk!^`tI zg%hDVosDzcEI4kON|qeC`^D#As(f+o*g?i?|W4h)Q++-9IXD-7+~} zM#^}%4&hsO)>^Y^3s73yWdS6pz?CR&qo9H&BVKixYs9!;5n*=g!|s;zPB^Hwb-as7 z1WJh{h#O6D^I0jullGRTiHwIr1;B|sTJJvvqI30(%Q)T?_O)@$^(hMgoUQdCk& zk*;2UoowS#*^S%i+pm}mIJXO@F6wQh$s@OOSNti6HHNl;TnM->08<7`^7(f=&1>wf zi-#8Lc8Yf6%ppyXeKM`arhc_Z&LXz&-&5@rAr2D`0zn_vU#YAlxg%8C?DT#SiZJN& z1CBPK5}4fbf;ap(?OQ#orCOoY29~VA!1l``yz#?cBl$OsSh<~PQgW%n2-lRJgJJTn z&wyk0_^pU=tcP1UOLGA+(4A*pYs{8626fstN1I5$ZsE0gi;Sg`q6*W#plEmLAp6rP z$w>1&w5@6L18dZhG}B6EZZj<|Mp@LX?epdIj#ZMyz%FJEAtWe6n(fLDO=~(L(T>te zec>&D({HJQYdX%zDo4^!OhKk)vLQ)b5$|OnCl zST}5~$4j7~6sQuEoWeSp8gtvFRsD-4?sL93@Fv@cF*dP07Z6383OTOZeXPooKK5Bi zI>^j4Jpt0PFkc2R`%f*~j8g94v`G!Lls1ojmHF1`eP)Mk)W6oD^shE#ukQ2dtzcc< zzlQ?Yc`j!AjxIplXn(F%w>dhaNHrC+0}$}f2)e^fx5eEcN8UM9Ob>6$v{CkbVTTsG zEMzH4G7#$-VtGjKT@}o2=Gp{#YcZ)Gwbm&&#Avq%&1<>HV8>mxsx>ZG>U1e8qtCos z_GjWOX5qIRxU)lPT2ufezy?Uts<6)lc-Mz9yM;GvXLV%rAr6-Aa;PaYlz^Zom^$+n z>3fbi!)z@J&p4oji8)eDSKtMce;^A0jieLCp%id#-=E|gkM5H9{dx5z%(>ULaZTM>r z7?buYRikQ1xKRZvZ6-ii>YUj@JhM=>j{cRmO~tTBlHyXNsY%R~0yaDA*w(Ks`*E`# z5PeFGIXBkmwo|~m+Xxbu+)Bz%GD+7lo&D=p{{Y4QAH$M>lwaLA*a0YJ#c-H95=b87 zepT1-+;Miy{5uY|q@_fuEv5i7uCO5K@}b6Y8>eu$U$AYq?~($Nm)4>P9Dtq3pPfU_ z`f(*-id?MouN(3E9BIS&ifquNIvZYg972+%01jvbPTJR<+Qku+LXxNxq10E<9u(u- zek^776>M3(WaV<*S~!z1LXu2_rqve_cyGgeR?6JOusk)wpaJt(LX>~&$_PEd=UcBK zM^cGZhW8eDAM~4l-lh6YzwcMD*Y;@OMJ+^ND*%}hbnJiqslWJU;9vg$VRQcga)YH5VdMc z4SM+*%#p0=_*WVnMTp$I${c()!-OF|NGmZU<|NDs?qtP8ZZOLy7TYZRL+-5py7Ska zL~9(mZUEb@ayvRBM^3rJygA3z3!FV*wZS3JghD|Ae`zt(_*E0$2>5#a;#_M8HcKW# z?QWfJ2HF6qkC$509L5d0n|)7ts%{%YLA_FTojQE0GX=*UwOYd6%uF~GkfOAKG8Oa& zd)2N<{VZ{{_3YiS-wL=}hp}Ic!|iTTw}EWue=7d0CYLJ$ey3Vr473IO48$!B2pCvKoU-bM`_-%o0E4Z zvdR#zr5j^>3+FJk+hQyffhtmztkWOf6c@JmRs<5PNWXD9exV>Fj$Wc>J%wwp<5xIK zWF`hc`$c77_piR_d?+-upur*uM2ONc63)Sy&EpH84}rrM76jLtTe zE*ov9PsF7}r7osY0Mch!8uIEZ7`L`wNm01kJ{v>!=taWNc2}H9N=TC+au7#-bmv&N z%r4!LlHjtI6&kHMokV@b8TW!uzO}DnTy*{ua)db3Fl5TqpH{np9VSI-{7K>i_}(DG z2}i>#C~&2;%92coCs7~5x?{?H6mm$(uPWKE9%aR$ zn>3U*L?mWDuKMj>XyNWT#O={?yd~5dw5YI<;T~Ix-62!7`h=c!(fDVM@%%>)a^mjZ z+S#Nh%UT;r3tGm9AUgrvlis(EHBHO28OijKyuWdW+B$~s5d%};*Hs^=gmsNaRqLA zl#xD?2INEw*V=3R@AAO2dIkIUgH!qcn!sOdYCFR*7-q=3+3JaSAl!<^y>7n}enwuGEt2`BNI`9fW${I>j>Xjr7bdpKh zwM*NTStBzKtj>vkJhD_UO7jzu3>yfWcQZXaggJLl)|;0b=yscO3k`? zZzZIuS8pe#P$pH0#bPL28wT)bdc!B0&inks3fc89R2WC$R?8Y@3fIwP~a@3D=+2 zd2d*k_|dn8gAc*5d#i%tkPw`{xXPsy0(;Lb{i}jC z;#U%&zi=@ZQ)uDL(iGa22^`2L(hU5hR>Q}5o0#*0%?MY*vY#>CAd*A@u7rI0*D&ID zOIHord{Aj5`h##+Blbu-M4jiAbjCj}7W-owmRZrZ;eI7zxRt`xtWd4w0}DDJ6Z<(x z1n>TO)=lReVxiX;b#Ri}0WJii(hicYT6=WTYevFxcP{~LNo}V=Coowhec)+2b*j!T z;svR%g+X!T=JhOpvyuBV&$l5p%WsyCO<{sm&Z&d=puAApp}Q9p$C?AzPh1Aj5_d?G zC0iJcs@<+7NVb0QO)#V)NFHfa`va-e_VyLQFcce7g(3d{F_jRag@c%;d31sZt-tD$4MC6?L3S z;|=(Xp-L{KWdxEULGMv;Ck5Z}WsBF2TpJuYYcQ6^cN`$eDdZERN%;d&@h1u5J|AKU z#%)5EYJ)3q&UTK2HdxgrOvcr;*78ewY^$Xah)Mq+eAT*(e<4x zQSldoze!lXeTFc*P9nD@>szG4ORFFGD?5~{ZFVFYjB-jpE?xVPkz3}!!M593Aumjo z>6`9E`=0&kE4I&=P>`E;LG>)aQPV*?K_u|l4SrE!x*(zKahjBwkSqFH^Ls6yF#1TGN*oQ-)C(q^N5fP7eyDwfAzor0BvXT^YlDw;NK|a^wuzvifx2sSr-}jf>dY zIDEg-ZZ@TY0c?pW_urtXM-teuatAW0GE|iyg%c7)#3~N{;;P(I?jLgI(5Vg->6Nst zD?HV%(lVj7%R1Hp2?a?>&Lepo`}3^;XqRnp=1Yawa-fo|z^B|1_ko{%>a}j!cBxV} zRPmylxN%npB1{D;NkARrvE^A7ILkj4n}jVsSOke*r9eo6HIB#U%C$=yYs^t9T>%CdDw`N-PHjXi|{UK*yIwe_lF&Zd?KJy~5Fsu&t&9?3o+L>=CBrnq9~SyAc8#=#lVrA$V_WBjdJt2g4C1+r~#0HIY8!n4MCB;R^OqFr5PD)VoqBC!Vudbzz}jI z%C_9lNFV zbEO9$Y5~n9O9Pisb)MpWbp_5Q?Tx*=YLyiTB|3s8a~`_$=k6$3ePnsTV7ZzShhY8_ z1b>f8r88P-Xq{nBxJZ00VR<7cIdqs2`OInoBCD}BoUlxlK8XQm%PLQ+Xpmq|nu^FM zORHMFEV((qR!NkNVt!whJ+2vhid-QHV7is80FanU3D;9FkYt#es@p8qsMGOGaW=M2 zwDU<qYhF$I7L}o1qy%IcbTB(|_O8>2cp&C0bce3-Pl9M51vP5WcT#zi z1E#{WZ2Sx1w$z~G7k4g#W#!zFs1vdP8pzY>UF#$utJLM<HN<&vRYE)tfN7eM*Vx%W#51pZM*8aw#FW_cqJg) zsUCPKfs})x1jhRB?^9tu6t9OdOG~^y{pC8@g_l&4m5q*VJ-bv(ELkvxri;a4?)hme zPn1>nTTmP7V8GTr%~xLGu7amae|14#l1P~5N9V{2+%4@~SS7|>$*9*&XQxUB+6)Zt z6iHN>R;4{XwwleQH={^SP*l;Yp_N>x(*bE2vXL|FdDgGuj{z%*IC1A21sJ8cFEq0< ztfN#D(L-+VYk9m|R!$WK9O~R=lM|th{$i*xPBHjPwA*=&mI_p)$$|&&#5Vou)8!mcf? zZQ5ee?5RW)1Li5jEitC2U81w{X7bB~9Smtfa*?Dn;CBMy_l(_I@TIc~Kma#zfUM>q zDMDbLkrl4-XMubpiaw>=;r5oONY%SHm2B}cG*U?^P#VGd*H61=62HMq7H+E6?lL9P zk(LSj!Z(d{>%CH8oF#~1S7;XOvx_wHN11VSypERvnImNlHV_9+b;oUKGk(_2UjG2u z!^6PW_WUyNEh|f^H+PPtD4>FcW=>fX-+HsO`!INAi`*eiV9zq+i5_8Y@$xp0NK%YR z`Tc9HTeR*coxAHnY>oM{%`VYR-yF0zob0clFL&V5|;(8WEi)7F-) zd99nJHV7(hAbNEvfCj3LQ@wdFi+JM};%+p*;tTtQ+}ynPn1!HZ42<4@kNHFZHLi!k zzAQWDu-)U%Tdc>HAgKf$GjyHI?YBzq&*Qw1qYOEC%Guan#$Lp(&Q=;*DTCyx85WVS zJf~oEo|N~kj9S9s4h66tR3Z`+3XN1jI{q6~TURFit8}2P4VY4vQc4NzeSs&rGgfx( zT&cpBLuCO#5KN8r%5?+lUp%iA`snu`8NlJVzlFS6!WMTImRAj9ITTd#*vuA`10Iws zmeLYQ_o}`aQ13&F%ufW5+Y8#x>uxD+qZ~TRHjo3D?!SjKWkZt+D_6d zPvS2J@cu9kUa8gLw=NJ1)-EAFVVMhXC_ypLPk9yV;_~@Rjzy!+$>SusLF}Tzc;s5V z+EH+}nk3G-k*MdU{Rb|!vf6Qhx=`~_+hBl0ooD8MuC+&k_z>Eyl$N-H@#PRwm4=%r z0Xu~NL7#rLnPuR9Er*%3$J=bEf&=#I2~awMbt7S0NyFUaQE8haTtHn=E)}!n0_t&oK60t^`%^NFd7kGVq&65>OK*_Iefmw6@>rvkjiMusC_ zoYev^4fXp=bqibJF_@=BX(TBVCJEE$yNmB|KoH4E+iw; zI)wm1=f0w#e!|NXwoSth`6z-NWRS0x`e-+-RbMrEaN^%@)JaKONY!13Kl1rhhYnS2n7C2R#&2ZQ@ z_b(qXTC(z+3X)`jvUTYdxp21vKBwDv6vm71ZRA?GqcN#TQjg@CwI!tUuZ*ue z9`T9vT>L|LelZs}R_~@1SF+ql*pmTTYb8L}s4!ew&xmYOd1CWdjXDyfJjPW54$u>? zE_+p?xFdzHAGSAGZKdj_T3zEs{HhvCLcs|l4xj^8_SUxcVGLWh;zx$8*g0{D*nwykqN}7M{g4tpNz1fz8)^=f-fXy$R*iWC zCgR;$e^%+tg#jXV-@fC$LyBPRTw1R3&?R=0`qZ{PY=R|4NZ&7T2M0b?*+_xfpDH;Nt}-C;Px zV+(C7!!G@5Ec#O0n~+CMbeY_aRYibs;}}~^`MzEC##k*dN>MYmZe}_FD)Vtl*oE1d zfg$^+NM_u+wCJ~l&&^-g=UWyXgV$DB~NUCO*&5J;-X<#+?RHbHKhi=>L^sJ=Zd}OOHlmHMw z0!KmDl=GU&zfx)XhL;s?5{9jm=^!MdEbcjKbdK89i#E?T>XxN6)9D(Zas=u!^Cmat zSZ&;Xk7X_btm`_h#{!2r*i4;bHK-i9D^X}Zw?=IL0PPQ;&!`O~oyi{i)g0UCo(r=! z;Q{qHUUjxsq~2cLug4^h-?BONhUj=>)NBlZdYvug&{eB%;JWi{{S&RjMi2aa@HI}&#f1agq0Ba zFq6s-;O*8V)VBQvR@t}k&xfsF+x%tf(x(!TP{wiqJi$;Pm4A0_wdrAa%Zw%Kc8`?G zB?wC-1tvsjd2Ar|t$DsIH!Or&IA$`d$qFPWn94g1MO~Jjx3jknE!SOuqOyXeCz+i# z(|T=aqx!Ddk%aLAEs|~90n$ z4z){W=ybNCwXrHHaY#~9rw(laJ@nVTU|e56ZEzfMqT@28xP%E(IxcsS%yXW#ZY`9L z6L9@9-Kr3_S13>H4G7ywiJ91QqQf|$<}U(@mE@?wQc@-}LDDz>04mqrvbDRnL%R8% zR&I#b~8Q?Cf?H4)CdJ;|91=Sy~d?*4fmiRsb1PQ>e@a%8kymrFGs4^)0m+&1>vqJm-QTy#M~y-Np8wcDj~RJE;k)8)B0(6KI5!0PqyMfx`_;TAgYAq03DHlbZKq7L0POBZjiMEE{Rmw31LQg6_YY<(H z!XCG~vsrEA~Towv^`rH;EDsgX_I}YDud+_L^s4@Xs4Y zBVaJcFC8HyB{@}-AfLLJ8y&U!*4K4z`idbcacVNJEXvem#&Deh17bVZAK~Et0LC=( zEZtIyh)Pn@&R$w+soz7UwcG3&_}g!a7Vp|Aq?D-zvR1WnFg1zO`B%?*xKw5HB6|KN zS0%zluMXoD4x?ERew)t?aaIY4T6{hwf}wbuXvn4;Q3^54 zYoO(z)1ej4multFdDfa}N>qjUt*g`y;ORQ@AnHjKn#LV>bjU)Oe5&0!km3m|WQ{-_ zM%~7hs%=QZNl8;TEKOGAI;&Q4Gn>#dW+?|`lh8=ks&N(DH-?t3RD`_Y0V`63q=FRy z4{lWrzB_T9v|@7NvVey|f?(+~3{OelQ@u*tRu`9vZ_T%PX;>&NT()DEe!JIv9Avb4 z(u|`1#vWT&^SFHvsksXK#E)*i)y4c+;58=;UOQy(k#%mOT4nTHUc(ZEsZ9*tl_=>4c#Y)N_~NA&pJiO7 z7MeV=+skpXkP!B=g=HCtI}$leXnpFA^NJls@%V+Q%Phe$lnkIq1fFAG%awL}zXLHA zZB*@r`fl4+nJ!u_B}v;wNeTImTZ++c{1M{|%;`=B7J_nuhmb*&xP$5&)#Jz1-MKZR zUroKXA?YMGgp#De%uhoVD}|;TQriUt9YGKyXoI+_C2;2w@Y%StcLBtd`zt7fS~#NE z8;Jme*ou$gaof9#dwYyG5w&A5&g1s3pcL~GQ>O7+{Hy7ZvN*nu+rwN1h4E$)ZFzX@ z;+J;VbhcawDfY@X(@w^>OgqC~9k5GmyKCmO(NdcZtwt1gJNB$@55RAmutV3mY?>Zg zK*&%M6b4y436MUu&|l%!ZKSzu?cx^MkjJRN-0~yytP`wbuC!8_|gXT-P{4LGB3!^~LPDMvNKYH6&(KKMwLnL7eT z^}N(nex`Flxi`gGwcxyXtQ_UNaY{>$Wk4yCG}eA{YbzP?KF1Ej*5Pe#8J~x@%2c9Y zgCE4#5#v7@oyE|bOfmbLKnV#cPGVE3i9NOhXsuGgx9%TIB|_!xLRP7f)k0%h^WI(Y zN1+@I$CR1Zv0odn?VPtot`w%$qM)Gbl9UPNKq6~E#dwn(;>=msaXd-X+iT^pnF()C z*a-qj?@(M_-4|P7L^#?Es4zWC?yjAw&&I0R)?P=$xD*zZpczj>5AqpLJZHFdKU zHyE%;0WL&ySf9?H;xTXCU;X3x)_>_I)1i9SMxLT)=S%dX-|wwHrmu1*iP!D!(~-dK zzPUk4qH`FSj(czZbc$VV)~=a!w&F~JiO$f|pCSDnDv6g=+i6mqg$PTIK?O=2_0U9p z4NdW@Y$<6`Xu(J+IaGS=1Pz6C`dP*H^)oSC5w0l17iQhM8@+U^Iva$xRCZDmr%0)-@kbWq%K)4g}RKNIpEQ;(H6!St0T3ZtnV2VbmJ zCyG2EzA(cTiQ7A#{@t}?aWGtXBuGi>M=ZxpsMfR-w@l-f*OK#@6yQ|kOPql``kjcK zHX7Bd*n?K+UPm%Nm+uu7a@?KH^{Lw>%WzbuNm^k`Le~W(pTx}1xvcATt!5SoXh~O- z`($_J{xyW6%`aumiz>x#g|wXll(*smM^GkaC-bYn(gxiqX{3vUHN1?vfvKLlO!v|& zT211L)Rxd%3NR9plpg(dHJpcn0<}s41h|kVDS;o)TIQFsHH+PuuHvN*q@%Y@Z^qslR-b95JAwFOL z46-*QZ=mP5J!-sZ7QS0E9J^ubGL(lBRHT%zJf$5FWX9jlCLF5N);b?DNrX+XSH@=QI460Xv%8RH+<~i zEh6cbmKtb+-c=BljKG4V^D<+vTIqHVFBr17b-)mpQ!M0?brY~Wq{!{wwVV&boPPa= z-^Ex)8|IJpcQ3XS0ObkI)0J=La#MgSAje$!0P995^Y z9B|7g(M+s4c8+{asY|32oq`XmO+qygPkl8Nk@`5}7NwWjDQ(pfEt+#NUYo5zZZ|x% z8^qSp{;lBd-2}Mp(v`VHln68P9FJyKiuWT@Zjs7Oy+%I>6AzvESQT0J%U$znLnxeR&H34Wdd~X)qcQv`PBw z-mZozgQoRrUKq!k1edN$jt{KmPT&mz*0U`9OgMgx@arI?<_=h;15Wvd`t|-*N#u<# zOwt%KJ)>H)#jc!v#_jOyrcl)7&eD!)owAKb_q~Kx$%Kqyq{Fyv-SY)twwF+(t*`l1 zqaboHIqh3E7vfc(FtU^_OK|`NqBcB%Jik4v{j6-hl;vsEEq<_<1Q_-^^Vdq+IU@f6 zD>9Y}{K(htFicsO8N0)*8ga6r=ULo38E`0#N>oWnJ&bR?W#4#q#xUo^-P>9ljG-m< zsAq`SsXY%+cB^OceGq)LG!&4dw%d1_g&5D88UgaDQ4n&CK%QF~v&`&A$BkK~;=Tj0 z^UJxlx@!-)SV>V(DoBB@n@Q>;uJuETeU&hLZM%D$yj+YyCRKl6%2KyGtSFq?e$f@y z5s4^;r!fugR8QRpN$H^c>m7H;6d0Bt+ynND>t3qnWDs&sMUB!z#InzVMuos z#W6(6NEd}Eg-mIc1DANMg&q*_-L$1PLDZ9|eZsVn8wfv#YR~;7r6BSmepL@AC2g|&c)Cc#Z-6k1OLQ{E9c^yN)CO896Z}usq-TaO zwu+LAbO9vv4BG30K4^vkFp#Wk<{%KpS%tS1qRPhT(d)8;C<;DH(L+c8};)WgkN7 zY}Z-fKZS2HZthTE6@|JMrvNdiCMUQxGO$OP3rjDwgiItYr4k0wA!GXITO%0C+GRfx zZb8^&1V*vWV0Wxv(_z)+Qza-Pa!CWW(qp#u3K8@YoU3Iy!Ckw^n};l}RN9Fj?KM>C zu5+)oVlRc5w|NWloI=(_<+(XY*F^mJRoHoMHoV~hQ6nOuB6I|p=6cpkR%EcGsnxi$ zkU?rjyvEW#l=(@In$y!L8NlpZQ)Sj3a3^)D?NB>JRZcI4w{qK!*|Mg27a6+p)`QII zg(Q+AQY&a`#g$6(G~#n+B_x7Q!e_VVQMYQ}a zf>hukKx?k2>!GUbP$iTjCAyKDRJ_nql$A#~)F+mmPW5;`GWwTha$KWv?HqpNALG}o z*?S4bL(imon7w#gsYZ1PB}y7}AR5{DR~O?{yw~Zgt8@Yi${Jekl?VPrfSzMh1}htY zY@2_F*(jS|Uv|7r7yrri%nBfWpj=6?tj=OTI&Q#oZ zsz#8H({1gUjxmR=4Z8b_amP?Vc?mI?K_C>B0j9AWs%BooaGdu^zP(MllvD}=a^~kb zjFZdptUIhO;nn43Y6^^@B;^J+GXs9LhVxf=eTq`@n?r6MZ6#z5zyl*mF+0{ybncCm zH10;V_JiURVd1;4+TAYUH|s+J2|-9cr4>Q@!z^v{uP|X=F>$vHxIPyfcHtvHi_h13 zNadPH71exdxN*1bu+`lq>u)T$lr()IWRQ1&PMXJR;tpYLwCa?l7XTng8XEUJYIyAN zyo9;aI>!n4i+kb$dF!->-L*&xc|?Q~e{>FGrCZ*mN?B9N31ASm(%!}fraRWDz*zmg zrXIT+ds}E_a$F)S~itqcY#DF;S`zBse>eaDZfr%bhG~e&_Bwl#l|8(+1x89?8=7R zss1Mu{{V+2KvswD&rVu9h-Z!zi)KAS4b!zB1*ag0(2*qy4Lq)#W&dHvfW-7T+qSet%XbX8q^s7_hCoE^Bu}(yuS&pPD9Y?^kl_e)r9dgQ z#F8?FNcSGZa;+ZYjNVO)pBgN1r@XZQUbsvXs7VO`sBWL@Z8GnSn8Vys9mWvafXwK_ zSy57tJc?P<=zZ%X_Fs*oceJ;_!Wu=^5pL@ALY)aRvNS6sok=23&v~r8Ulrj_9A#_$ z9r129lCp+!l8>5%1b}5Jh#dP? zYn)@@vMv;-U3S;Z9;s#2Wu4@LN8!%3YY!XIhB)ZM@S7|-Bo9@+suFeiNgHolUJK)1 zACI_j+-Zw-HcC{NofyZTRual5(%5BxG7m~6+^2jm)Ga$(Ar9#CO-$Xo< zqk3z?91VqrZm;g|R1^X75*q}bo1T?AmbUrZ!PvP$SOs@PsZ$z6=u946Yb%4X z)xCn*CjQk3xl2i(P*%M4+?a_x>dO|rcIw?ln}oJt{nTYq<_@1dhf3#`IIW{d<(Etk zjJLs9LwI{OR*FA!$!{ck03@CIeCmf6;%m!>ol>rkmjy(jWB{I9%s~?(WK}LJVTw9k zIpMNQsVW-x&~Mz<1|4a7`W0@aDdyW!b2~6Vou{6kvs^O9)deG6T(&CSz#MTN%{w993z zJ%$p7%&3!;u1u2&*G{vn>9MMeUx}}Hn%~Q`c{qLZY6VR+lB7D2PGECwIzf`7MczbYO8+bLpb`M z#MHIiw1u{%A1uhYE>V!JllDyJ(gSTf`mBYM$jrMrfRCn^*dKVFC1^Qay!y!Z$K zOPu~;)EzhN{uQX+mXg};4e}mkS}hHxCVyr|x^*=zj}lo3X?F`+kD&QwK<3^I?jz~* zt0}KkX5jrgHl(;3GWiHfPC$}C^#FAOKgtk<7I&8aRScCL?3Wr;Jmp5{lrR9iZ$o*;STEC!ix;nGszImv|( zK%HbD8U7TGt&=92SJ{1rHsz7>TT6i})TkyDqs*v{$vd9IlvR*b$Vf>FQdBf5236X3 zAJC{0x6PJ9v;Yz_D4wTp;ruEjp}?8A{Yo4e%S8mlCSbQID- z)2Q=F2|E3GbDEXc9xf|DtilKea{W2SI;Hr4(JBlE%Y)MBlRIgcnX3mJDncDlONvkg zWl`z5>T1!I%9#>Mv|>5tB|%GQwF!{~%6@_gqFceGFE^TlKkW>V2K>6~y=2@rR9jJS zCoswowU_{%1RamHVtJ{|q?mER={Zs$b)B~cHK~_(VQmpzV!swr8+B?Wd9t>UV_ibO zsHSno6r9RXa^R|ET1k#WZj|dOXl4wh!Zgs00qy+j5%t@tgoA2Uguzl0qE?aE^6UJ~ zOXWsttY$F=m{MH{c?lB=8mBUJ)bi)4)0IN@B-O*@dE}P>B~Bth9X0#0shxSzs@~NW zt|cy|rq)R*aapxIzpX0=;Q|XFJjp@LD=iYCJtIT(`PEk@?9=FIWxShockC5hgxl{;r<)24JwXCj<|pb|&p(y~=$o6?Kc)KvgxNl&~H^VXm^dfR-h zOK)FJzPZ*kox1#{YHhuuDn6$_ZxOlSzAI85N|?9A$$Mm_w7P!!WhCliVjz-hpfki4 zD??AYZ~`*OTkIUj-T~OePNWS!wWj!ofxC6XZ@GuB&$+k$(#jGftfNwpdX(lLI)?~1 z7r1K=F?3!^(gIPK5vcWL9sT)IeRF1u`c2aNiR~?k^)^=LrAk34C!k3n$OBzX4W-3} z{{WQtn)9TBQnD05J46s5>G;;RtS;^G4mS6erGlc{-db1QHJR@en{E3WhaoCL;$^jZ zglHs!L5{|#r6k?067j~8PiHavP*|2Rj2_{p(7SIfX!F9eWppSIJ9>2y-=%Q3Eyc^p zSb1%XyT{{Ri}5o8e~CDD;kEgfP9ytE30X>1mjKi!K>&%+>#cdt{p8`+->z`BN`M&; zqyf}dyW+`8G`$Ze$b*cn`j^}$($KfI3H50zliXJt`2PTh2wO^Dm=qL)yhlAY<)wA555D-^B{$co32pgSRdXrr%yW*_ z9zi~ z^{QXet>SIA<-2}ZDL!YIjIuU>IsB_Jo(pRY@||YjsVV(pid3U2KKhYfblg(vnb{QN z^z9jLev42c3n2uJasm;ii2!R8^{C&ux3L>aAe7>uFRY2v(^X27{G1 z!z=zFkNX8MzzpM8D0QA@U90Jdbaep_@{UBM+W*b04 zKsh-<-<45FUwD>I`g7B0CY8}H*w}X3S}uw06OVWmE=el+)QlYfzx}>O7iMT@`B`L}d!D^rCdetHxaZdxf3>>ljX;P_ATsjC! zV13-fdaR&bWZu!c;T|I2;cb?fytrYYeJMiDpilEXD+qDz&9*MPmYiL?7bLdn=0Q5H zI`!*Xp;L@3l_C?d1n5m;U8z?pGPwtk)0J64BRMtfc3Zz4akV^#ZAvWd*hp;-Tc6e# z08FYz^Qie&%Y%6Qi&nq2sq4pzWhDs8l#wGTJr0`d&TGyV*AH5GX~}IEx6w?%bTpg((UhoI4G~5G$qOuUcH%rQAKV z+%lk5p50l{u3L>iGqrdo*5d6fIqD}gB!Z|gIo8*~{AG)9>w$P>>$a;ZK9&`ff7CWR z4aITf@qLbnWv8Rr&f#plUU5x0G6Jv?;$VsAxj%;0LyB;{!>c8r<7XrxXh?9SQOs#4 zU5V61Yup9n0##*bakAaTZR0!D*=Kx;g!r(5H=ZYUD8I!awmgD@lw0gyEt zpTf9hfh|_fmE=paRKsw`3R2U`QBslumV^Yz)AncDD$U#GaOcx*?ZQIIge4N4d+Yuj zs>bIFutnmYQq^wkCQ4&&QbxnSwP81G`MP8%rB=j{tmqOX{_&HiNsr-_aqgyF=6%4x za>Xs*I*W^i+`XSLWt7LOT@q(NM=zYz*U1efDq57wj)VH8sDeP9K_2?;^Qm$6P!`gX z!kJ|Nl{oBAJpt$R=Tx^hN?ZP0SI(n0qsv%ImO+gGp0TdASBu!zIo(rRGF&LOl&M3^ zRFx?pu66t3HG(?qIaB_yFsE9%X$UHe$r+T8bc3d!{xo?*C{c3ZEx4UY%t$+C1Zk&Q zs<}gU@*BAFP)=;Ep)fi?JIs^vndMlxea)2LshW#`r75?q^M;ZOo?=M~BYdhHwkDl8 zlNr7ZIB2#)mJ%ExK`Ma(QbZqq{OYyrWDAu1E7{pP9LXPgL<2h+jn6vI;VV?4^~<~R z<4%RbTYF_VHxs!cH#;7>RH(k$4r`=tm{tz(X$W1jYRHn7-E;4NCvH&(da`2WkQ->Y zrd$FDE7AxW=x2~5RXdzSn`OAs^HQi00Ft6co|Du>7>KM4JpPqZ)}j*HR6tq;I+YmB zJdWq2RPn8&K5gln=U!t}Yd5 z;X+(qVv?N{N{E<^$eGt(^=WO4{6wjw2Qwr9Tm&Quf^`x$@3mUjsw0e-PRyITR@Tac zjj2u$HDO26b&~^AzOmA^E)cT|;;tgLIl)ECBsP#rl#mCzhIWtW*toTI7wXivnL#fzC&*ITS)vM?g-T z^cuxA)$`Wt@=6>+QlClMH<7n4{{Z1t#b`=gD|s!d2c|&>{C2EIu_P&$owvKUPyiAR z+ZgpsbBKuVT5+#KH!Im__AHjAp_xn7YV%jHjf|NzHhB9>X+bSF5P^B*{ARgLKxx|)hU?8_1s9JvM2Wj`;zMo77oVRV}kmIR&2>?O#tGs3>n5=ZT z$VpNh3Mx{w5&)0~Qg=GR8^6q5x<%ha7T)R~ZeN3~Sojkm-2#wBX2r5`!bb8e*qWwMlrBa|#CXnBg>Hsb8X zxW-D2pkf~o>K4*VPHtOt6olw{_1{jlSAB?MAl)CsIJx1epA{tQYQxWFfSr#0`KjK|1$?8teE}Q+hASc8b@rg)IbtjGK^BQc2!* z`B$4b?*aWC#CPnaZTSU7_R_v-Qi+dnL9amK7=^v7VJ|vEi$1ST{dL%#IaeU@F7FG) zIE9Zj)hP}z|hFkex#Gs zgQnV5{LGt*OwY!L(%VKg>9oKt$HQ7J%&7@UxgeE*cMTa+N)f}m5VFNb31PZg{Zo8%O7?k0NSfRNil13WUXmTq^3qitC^qO+PwR1@3XQg zDoC(@7h5+)3t*)IDpq8A)Ck>LY-4J|w!DAOecLA%=s_f)XtDTyCW__Je%oyarE(Hb z5da|cJCRJoF5%bK*9y0FN@pdMC;I_rF z3pF42g)XFZ{gL&W*6zN^7w#0BE+e&hI*=T+QgrVMHR(%KrELW`+S>h`lQYxlrrm0P zgttQTp|a}8guQB?Z<5yO%BJ{$CtcchN>#sWA zK6KWaGM|f&x;}Vc@IMjZPJeB{@f%eWB)Mj|KQIWYw{4tdaI|>}+FbNU{nhlL{2wwD z5aU$h1c(tNkKtDQTqW9NUymi!k>xw4l%x}vDOagUI*I=P593jr6e?1nK!_kg zlON|@=ZwC~m~I_pw=ljU#ui4+E#x{09R^T(*A~V&GW!R*N*l!zpai8jgAOy8&VUk} zrgalMt9*GOk$o~TlxGLDP+|-OkeHN?x>hBliAuw2fCgmh+1QoPAlUuTgJ7J zqRPs8@)5DBhw%ZKNLnFW!Q6k1b;UE9U7IQplBESKEjo==xAmoGS}iGFRiy3#$~jGI z7=A0EX$4Cr0<@a3Zs@kP6sVP`PIVF_=pwBszf&~*jrpuqqS6qA0=AP1T9>IL{^_fp z7x5DR01w1au;-f4FhhqhN{>ZZ`LerWx2SduY>mMB2xO*a*_=Txs^{R0C(*$M9pol-zR|K$6 zqh4`GeMVn~3A=Equs}*iqBfc6IvT3E!mXM|n9q=xkaU?6M2QkK-+p3iQo=k^!Y)8& z`t7x5KuVGpB`0tdCOU1;m2A!3**el*vs3QdWg#iem=b4A0Vk<3RGeD$Eg2{3L2`?h z8Bnn$2`5yvM34shjr1e=RdaTC*VgVK%=Ydp8qr47Vz5pgQT4`j?eD0l50|)RV+z3TKeQ?2E%wTa z)N{sOVoxGA+nsDS*ETne21~75-GGpmT~DB80y2{`5v^lmjXJhlyBL!;XxQ7fz$!5$ zZ^{on^fgTN!oX9mHiaQbBzZ;_U=C!=K|9r4TGC~duX3{0S9d{ZQCjWaXaJVfN}C;W zsO{JS0O_qxn}-stkEJfO(|Kn)N|Vf$$3K4cMxG|Mu?- zIE1#efEKc%h$EQxAJMKg#C|4O@Pbqk8D)1=QtVw^zDy8$gee5S}q3R)CYb2G`hVgS8QEc+k5P0-RN;D9$*Mb9*K>%>GSDY{JKpT^KGr2B{K^w#wfxS&~hOHKr`6~(p zmjuG3n9M-bD43n~?OK(E{ljm{m{Qt=r_KP75ANk7P4$ku)@!(qT*bv9{etBrfT5>N z^w@RO9!;*+$)x`PD>h5#5Ys}}%@+zm3R;zySmtAWd5*gZuX7bus*7!<#=z8!#7R!F zPcB~5tzorz@k&dm$k_m9O0-<;eo2`<^s39Nt5r9Ur9e5B78jKxQf3D|_u8XnwDio@ zvGH=OtI3!^T#`JpOn_1bzWY_W-F3zp3VSJE>kpR;j#&h0{t@q5ii2u!x?NFkDv}V1 zR;Oq?jVJmworYq62!yur%!bt<$SRoAG3!g+FygAwnEocVo*rW*3L2-%B!v=ZU;yu~ zzdFyg;#buSW+h@XEei)Q+yDs&ZeJ~H0q59uXDVILqsj>xVD1Ujoe1yrsGGmBWT95f z1v{ZCQ1$ZtuYP{jZgh;?E!k%FCcA@hmKbcE2rnN{N$00BJMxOIy~V;<%$sC}rzwRM z8P{Rb0guGgi@dbDw$M13T`1yy^#qzF?-@0cj!>)O6J|%5~pK!*~2T57E(eq&vzZr)2SxD@hyq6%{*U_|RP)A{y_q~X3La^r_^%cl~AI$CaX zD>AbX3>9t+=|4JrTK$`sJe;jM+nN`UMo^$W?|sgr_|}n1`zYIen3$95butx^rH}+T z3=F&LGG@6K6E-m{diBEMm*u50rTRc10nfgX-)iXB7b!{dONb6Y6B$g201YGjE0l42 z*I0fal?M_UchI%Dxa3E%)9GEf6H+nBB-V|Kf^df|zh01}rDrsQ8cc7?ep}YxjomW5 zM%@c-)WHr1TM!om(oad&n}t6{o*%VoD-XVw^Cc>HASFjqGJz(uuTsUz*-Pw)oKXa= zvI0^7m{^@fb7g$;b+c@LL~GE86GIq+-C9{&tGI%+2`B@QB-JzgIJ`#5=WaaKGiVZs zkO7VMt)=UWd?m+RBjR0bCn`ceR^LERGg{TP{nh=&%2ePEsR}4<*CgfK#+nMGQ%>Yv z%GE@zbK)6WSaVLOp&&|T6qzxpWq2sa70q*3yvEezm&!0Hgt~+HYDYtrc^0*-$O7ND$?VLy`rtIeSdg z{4MwVMVJ2o#Xruh{{T+eBhriOY=cDlIdY}?e%Jmw+y4Mgufdso@6_t|7}K_jKwB3p z3RDjzxtY>WFX)|Ww(E;gY}$kCETtfoIH9oa41Yqh&G}n_3xwy|H8Kw^Ya;5-*7M6y zS{6V*e`O7}JJ*jGquZ3d5m3(^r6^HhrV0v#W>St_2E5} zAG@d;9Y`9*VKc+nb4m&}iML23dJ&Qa&;aGrO3Y^p!;}g{>je6QgB=e#+N)+QN2bWJ z4luu2Ti#%{Wl)8nm{1TzjXLR6V;z0#l=;b0Q3RD0LChjV9$(hAcE=1`H11Y00#4~| zL~42HcN?C*^=Em(jNMwPE6CzixXb%mdmY7T7M9tQkFqrbcHwbK7p}5`4nWc~$l4CN za`uX{Zxnvvmdclr#R*BxuWZ2Y_!jVbqR5>+R_$< z5R;Uor%t0|ruA-6A!Ct?cXIN=GYCSHqlwgMI&Y^xYQVb-acq)H=twe`PnZD{*K@v| z>vd-iZHEdL6t7V^w1YiIKl1NcS2lKOZ!2uDqDW9l(Ek9PWaO_#kx$g;4B^%xE6rtW z+HL{N<4q?j%xfk(4UV->iuifMILT|`@Mf+`R02wZmfBP744%6EYpAJ< z;$UsBw)HA(EYP$#t9ElH2+9D~G>MAO=E*ybjd-tJoCfpk-GJgQ8-^^NurWDoii93u zg-DHRn-u#qBH@J_Tz=uD9ReO{wHS_)lU|}0R>A2FlDOC|GMW6O@}srE?Talfz|)H# zcs88|@%g(l>QQ*TZIG!TKKFs`6W`DC{mbaUp=OZMU7y{{V$`L*hA? z=2mk{!@&OlE2HK;&*?lSMdge3v(46&-{vS2juI1@6BYDt5b(%cBsAr_FEXT*mfS{R z?X_t5=h=^k@n%X>gff@WGHnr%f@edZp4Hb6h{9i*2RytQ{Ifh`XT)gC(OZ&J_dl+g!_pSc`h839V|MYBGx7HUq$uT9W2aFaD(43Adc-!{ZKx{f zKq_%$Pa*6zC+l3&^TEt_33#IB%JreiOv9l3xsPhAz2LN3$vB(##sNPWXru}0BCehZ z=`$ybEO#V&Ux#?n)wXV~ufvV3pDZ%wWS~y*(Cx0e`&H7o%H1sIzCbEj&ZV{vPyqd+ zH5wg=gW9~D+niB@vVTY1qWaV%61yin)DQ-nZ67GAoM(u5g88uB;%*skGZX|Z0Yi~J zYCjB{)6rgXTdkg#;$IQ?M-5@OKS_K+`>1WShZF*Ifd)j4b?02W8}YX2xLUxhSh8#o ze8MwkfdJ(rL-XrgZ{n6nl?grdqWH%WnEjq&x})PpJEJEz?riv<5n@=q`6m_|Jl4 zE+wa&_;R94j;=(g>~n3rSD8FhQFe}%CCZaG>_OKpT~Tdr23$cILCTaQeb9A?)^t4U z8ph`Frx2j)jy8>Rts0#`h}86x)Kpe(Ch4NHA)=6=md<44<*zO1ZjhUI($%YLQWSpk zPH14s+<->>w;t8N*4MG!8?MX7F;avj!h~l?SL+IfWR5^|Kg84`-M5K!TqtJbg%uK_ zKp&6gHLWdo1IlRuC1pxVJfRJ!#GO;7<_@2oTUzq$PHPs?BPqq($c+m)@Wg}0b})wUD?iGe$F+pT0emY!K{7Xihr zYLyuTbp~|bUwW%m`LHQzhLDxPP)eR8#Oi~ra@?NXDJ{EXM_No9vv7r_E?jviF`Y=1 zsHc#FyqO#Gr1ijcu!fQrq5C94QwK;*O5+H;FD1)Gd8q&+%TNU%PULISITKjTKk(~H z2Aa#BWDuPPsGWsH(zIP$9^c{+mlbdzpcN>n5R(u@fzExoZ&9}PS5mc?2buu^lc1m8 zNz|CjJtCxb@y||hf|ldU8B(+XFjP+C+ob|tO4!O+LK_Lpl20wfa-wR=w9%fGd~zHD zCSTQP60nJqjL%2N^!W##4fmGV0edgok5q%2$?Fu2}KfglnTM>sM@ zkzE@R!_xZ*EHqS9b0jE}1P%I+aniZ>87tu~MYqIT`KlxoqzxxcdL1L_?Oj-+p3#J* zT4tGT#`jj2&8e>~4l)1=Ny<)|O?1o;6L%Q0RpX)eSW)v8=b5C}Z9+OinJ6z!zGl(2IX zkxB*yDl#63(mbz*cp^H+(+Rxw7bN&SJ+Bz7PQ*1U`uV~ zT%Dvwr@5^X^5@BMwt^RxVoK2fKs$9Nt+;8&Tr!tdQr4(x!I$4%{*-AisE^L**#(n; z0r5_b#d`i!{{RN({8OLz{{V$JwY_j>_;${1XU!&`;IDt-+y4O6H5~r{?T#N~vEGEZ z$C|d%;=!0gy+m~UG@`RZE-54|qIv*E2||bL z=m6^n>0SeyU7tvxx<#+(+iS9?n0I07E0purb(#TU+Cmz20cueYq-vmc6Ijrzg0F{t zO{{_^EKZ^(c~)C@H#V)4c*}0BDH&GSBUAnpT``*VIOL?dBt{=_lju%#=FCjx%6g3| z^`>rvEiJ{1q!T%iB*5lJKAKfko&B@OTV=->%47iMnr*brPP?nN3KMSgPclG1tp-V+ zL>cAhxPoK(A}S~m@+I)s!3C}lCs{(9?Kw=Rb~wIrm4dQ^}zf(X=&V0`u3 zvT?6dXyk3}8rT=0r^>kahK{~j)IcU;Bv0c~yL{`YLtaRf5t~pB+^1u=*0SF6E#D!f zsinaROHdI%y%0a6QY~pLXFgigPGYUI7$8o%R&6tDOvbT&;VM!VQVCvXBwW3?4Y?a%% z8(UKWM5G-$bmgw~ac_RxCo_lzT2(w_AFI%>D;k7B%`I90PQzAAapI&21o+&D2 zdC5PgMzMdxH$f^(iDu!oN>lLYl);W$gTMGzvx;yHi-#-Nb8P}4OM;^vKq9x--@0ou z+i<4YAj(pq>fDe@eqW6R<}`-wR#t-g73#{rQAvXu@7S823B4AMc}KK){{R>8TM50r zTej(L+2TP%%%3e|-9vC=Vhr@HGZXNe8^c-%!|w~N=u+E`=1!cnf;o=W>eWAm-2~@H zkm2Ui;$x^Fjr-MA{{Rl#o4}^;3rbQ(WxH&Zl?`LnCO7;mNX~Kf4da)kzwiG52b5OD z_YI{M_`eUeh}#fSwq0otr?3hs3In0mqqN~J3*zOVTj38`T*s-KC1qTXyga3Ar%|}+ zUZKS}e#+goWw>hQ>>omqV?hxh>GS+6Sj9LN#~4SLyIXR-R#fw8kv-#3bTA^dUQ1n- z{4$gJP8!#NSUw{L()BGirDZ~|Ss)!JQj$V`bzh8qp6={Kw!=4RAmv?~>0HMCnVl=B z;(iXjd?yaZow6X9Wi6YO9d^q)0q-5Fc;PP$@f$obKNE1QM5$Sg!mWc@2PjEd0<>OQ zO8J$3H14>nJePIh<-Q*8m33h6nhYr+YHeRB2l-WX*N1Jpc-QFnmK$2KlsZg^ZR^?OT=yJCkOuk;YW-X(pPTw9`vNBC(TlRTNpK7=h?r+yffC`3+9QK zV(A8d8urziEv@{v**7n(Cp0+$JtAYMooe#Z#?ALm)Gh~y8$OJDMM zPSJiT@H$Y<&I5}ElMvIEh2#J>0wjU#VzN`>Zx!0EZ?Id$Wg51uJmue*DgHI>%bZfp zMJaWL+Y_grg#u45brUxhNJ3P%!wM2ZX|-9>HHlE)zLh_UQ`^RwABIrwRCxaY1p7y| zWS2%OYFkp0K=^iil1C^6fCTRv&pNcb`$=GEIhgZbKtv_Ago7{)oqk_kYt%T4m`&S6 zxP?C6iz;yqCoGBCQh%?VP~KsjG}>kS9@SzA8IXqvJ8A-B=TXlbov1(ZSLN69{>Pk$ z+7`m@2HLCmF?3=x>EDuiR2{rEFWeb_M|@I>xo!Fv~`9 z22aD=+gmczfhF5}Y=|2eB7I*W^sMCDJnpm;dTF(7$l+V{Hm2_u(@O~|SDNY*l@-4o6G420$Oa8loGArs!1e{U8aKT6Zl6%D7bajl1g$VI*m-FP)whf z(koYeX&5CLZCfwx_%DX=dnd)k{w%N#VgrrIN#8(|JpTZoDu(L^;k$yjgZPc8Fpyj= zkVO69B1WV0sBdpvyCEKDlA|z$9Kk(F3W7&)bdYNRwP|eIiGK3jgh^5!buJYY>;Rc6 z{qga_-ekf&ohb%)>#U2W|QG)~i>Kv2mGJwu`v9K}ca&Kq)$n zNCE+f*q*hl9Gkl-B$QfFF>kyv;rwmn`53-4Zt8?77VmN=Af!61Tm*R6I(B-^CUR~W?GF+UFcKCyfe!dPwU z%8qoUDj~3S-9}&zlUiRISYZ4(!!F?XU7j2g*;HB}00wPlBojJ8ropjA@d&lMcH=sT z5~axmAC};sZ(&EqSZkMfd#2q8eJu>+IIO8cd4vtlq#oqg40FxptglCW{B;~x4$Ud7 zN^Qfdxpgp3PcM1gkL6U?kKztIwJmE(Nyw;FJ5IcWetI~_&`aJ5r* zJKz4&=l$RQH6JN^mgLU1%a;zLH-bbq32Mk zoJsNn?=+rdI*6a{)^)ZMaI`P`Wvs0t5S0)|O@FRcWhp+)oH57H#&d|SQrbcYX(^1U zB#k!uRyD>VtJI};4|#|L5P~#9q>Rby&t2-#yd+zsF7EMy*WUeFNhzL0m8GX{5`Ih%>1=ley&@Y5sIA;xDIEVOF1cAO$3( zgP;J+iPV2OisJJUxkMt?&e`Cg6cv;S>^-?al|n&Ml%9iG*QHe1vxqGY3(PLw;kbns7Nn{=h|n31RIh9wwGLh3 zz%sF_1I&`(BmS*-t0gFOQT)7`Zkct9aK+mVqUFV2GSk}>u$77F5z0B$Lq83MvK+a% zdg&!4DJo_YJ%A%f)PiGOw1}+5aYN=CU75t~g7qj) zuzL-DM8#b1OUixB7txfp|L zB}|swErLf>oj#w=x^4Czw^Ef@F_DacN=lo`NZ%}TI(=fOv0ORJhtgH%wzm>eP&A!4 z%mhsH-nW)VElpD?2tqFL?ANf4H{!c1ON)z~I_0FctwU{x0s0g5(z+iC@y-XtR#y3f zjoaUQ%veEaPQH>g9+=W=o2^$?uR7n7tn-;UlnhAcuH5(f)K=?y>fs7omaN<r9kWy16#V8;ND$A!bOxC%Jw~gYqA4+FHV26tO(k<q5Bu}IeK+*^?uCZB)#e*qvK4@ET zeNO<}eMkpg;OpONmYjbuhZg>uGuh)?=LrtH;fYXENd%I5^Q`u6j|xt>&+ z=5>#1!ow_WuI|7~r3FL<1?6ySp(AjryJLp1+b0%ZOqrA@NeZ1adPcyVXX{fG`>QLK zP)fZ^$1xV~8z$b--_&JLN0bsB8xRQ8@3$(gdhY4Q5R}>nQu2z6YIKD)9dwz2%b65S z;g4I}AA!5b>PRvM`b=x{+ox)$xpbRP?Ju7+D6h-T}IWeasLv4*-sRS^} z?iS~w0nQ3XxYTHLFui%kUu|q!FBvZUK@L28$n8fP=S>> zqf(-EpPxF$3cI##)!x3-&4g!%flj&;ro88FRlm=CYt+N=sZRX~@eV6nHm%#?wl8^2 zoXxHBtf?IIPM^nqwVuhjyIW@H*|dUEP>jbwM>GIW%4-7H+pftiytKlOVw{>2AdN?E zUdD>v**5O{LJC_*I;{{$+{|;_(;7|Ndm}cLB+L8UV#^D&D{|$>6riM~Z3zGkNYhar zCt9ztkGAVJ!D`<=dZ-Hs2EqsqwBi?dF|IMr+7X0YYq%2$?^ti5iHSLz#?m<1sF zq*U)?7xx!gRdI5Wn5nepNb@(;5}gj*=YF$V(pzrZi7^Y7(Ukd;rxWX5K%Mu{@1W~j z0&d(=Q0t0T`KLa7#Qy-z&#yXXoxK|P*w-JZ2Cz%aZssmfmsB;(e6TZv$Uy#7*6v|AXj#hU zO^{Yo>VvcmL`;2X-@9%16P?8&qN0@)44`kU4R!0%v99e~Ap=Jg1eYKwTgV!JT2 z)W#{U`XIW;o5cf(N}BT9AxR*ls&x_5mtJ+N@m~)wi7-3M7GF{`wos)Z+C9*kWv9H0n+~BCsdsdo5+F;#=2K#;#M;7&B?cEhzS7&WXxzu)M|Sg z=RQ1e+N@h+iE!3b4T(Z7GNdM1QHj&{Odi0R*|;kgPZDsjUHZJVA+2PCo>3Q&iV@+K9>?Q3Qn?-~rkO^_7j`=L^Lz zz)Igt1haIs!CGd24z*lxr!yhMGM7Zmlc@Kd>)4D_JefBgqR9E~p&{o~;^Z&OSt?I$ zJ*km|T(;o8FZmB>v>{8%r44@Q9JH((+tk=LzldGHb!k3Y8vqR4iJn{50&y1)>ljW& z1caY4a-up3tEe^6RU+Y7s!EAdsY<6&okl+zZ_DtN Date: Tue, 29 Jul 2025 19:08:23 +0800 Subject: [PATCH 2/2] update benchmark --- multimodal/tests/benchmark_openai_api.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/multimodal/tests/benchmark_openai_api.py b/multimodal/tests/benchmark_openai_api.py index b404725b0..aa5e520c2 100644 --- a/multimodal/tests/benchmark_openai_api.py +++ b/multimodal/tests/benchmark_openai_api.py @@ -44,9 +44,9 @@ class BenchRequest: class OpenAIAPIBenchmark: - def __init__(self) -> None: + def __init__(self, host, port) -> None: openai_api_key = "EMPTY" - openai_api_base = "http://127.0.0.1:8000/v1" + openai_api_base = f"http://{host}:{port}/v1" self.client = OpenAI( api_key=openai_api_key, @@ -269,6 +269,8 @@ def print_profiling_data(total_timecost): parser.add_argument("--image-nums-range", type=int, default=1) parser.add_argument("--frequency", type=float, default=1000) parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--host", type=str, default="localhost") args = parser.parse_args() ds = load_dataset("json", data_files=args.prompt_file, split="train") @@ -298,7 +300,7 @@ def print_profiling_data(total_timecost): image_list, qa, args.req_nums, args.multi_turn, response_lens, image_nums ) - model = OpenAIAPIBenchmark() + model = OpenAIAPIBenchmark(args.host, args.port) global_start = time.time() @@ -319,4 +321,4 @@ def print_profiling_data(total_timecost): global_end = time.time() print(f"Total time: {global_end - global_start_time :.2f} sec") - print_profiling_data(global_end - global_start_time) + print_profiling_data(global_end - global_start_time) \ No newline at end of file