From d2183a8b323e9a5f66835ec9df01d6c49ee90fd0 Mon Sep 17 00:00:00 2001 From: xushiwei Date: Mon, 24 Jun 2024 17:27:12 +0800 Subject: [PATCH] c/llama2: rm precompiled *.ll files --- c/llama2/README.md | 10 + c/llama2/llama2.go | 1 + c/llama2/llama2/llama2.c | 2 + c/llama2/llama2/run.c | 975 +++++++++++++++++++++++++++++++++++++- c/llama2/llgo.cfg | 6 - c/llama2/llgo_autogen.lla | Bin 26554 -> 403 bytes 6 files changed, 986 insertions(+), 8 deletions(-) create mode 100644 c/llama2/README.md create mode 100644 c/llama2/llama2/llama2.c delete mode 100644 c/llama2/llgo.cfg diff --git a/c/llama2/README.md b/c/llama2/README.md new file mode 100644 index 00000000..1fd624a1 --- /dev/null +++ b/c/llama2/README.md @@ -0,0 +1,10 @@ +LLGo wrapper of karpathy/llama2.c +===== + +## How to update source to llgo + +``` +git submodule init +git submodule update +cp llama2.c/run.c llama2/run.c +``` diff --git a/c/llama2/llama2.go b/c/llama2/llama2.go index f3067cb3..71a19e98 100644 --- a/c/llama2/llama2.go +++ b/c/llama2/llama2.go @@ -23,6 +23,7 @@ import ( ) const ( + LLGoFiles = "llama2/llama2.c" LLGoPackage = "link" ) diff --git a/c/llama2/llama2/llama2.c b/c/llama2/llama2/llama2.c new file mode 100644 index 00000000..8a93d945 --- /dev/null +++ b/c/llama2/llama2/llama2.c @@ -0,0 +1,2 @@ +#define TESTING +#include "./run.c" diff --git a/c/llama2/llama2/run.c b/c/llama2/llama2/run.c index d37d4607..2fcd687a 100644 --- a/c/llama2/llama2/run.c +++ b/c/llama2/llama2/run.c @@ -1,2 +1,973 @@ -#define TESTING -#include "../llama2.c/run.c" +/* Inference for Llama-2 Transformer model in pure C */ + +#include +#include +#include +#include +#include +#include +#include +#if defined _WIN32 + #include "win.h" +#else + #include + #include +#endif +// ---------------------------------------------------------------------------- +// Transformer model + +typedef struct { + int dim; // transformer dimension + int hidden_dim; // for ffn layers + int n_layers; // number of layers + int n_heads; // number of query heads + int n_kv_heads; // number of key/value heads (can be < query heads because of multiquery) + int vocab_size; // vocabulary size, usually 256 (byte-level) + int seq_len; // max sequence length +} Config; + +typedef struct { + // token embedding table + float* token_embedding_table; // (vocab_size, dim) + // weights for rmsnorms + float* rms_att_weight; // (layer, dim) rmsnorm weights + float* rms_ffn_weight; // (layer, dim) + // weights for matmuls. note dim == n_heads * head_size + float* wq; // (layer, dim, n_heads * head_size) + float* wk; // (layer, dim, n_kv_heads * head_size) + float* wv; // (layer, dim, n_kv_heads * head_size) + float* wo; // (layer, n_heads * head_size, dim) + // weights for ffn + float* w1; // (layer, hidden_dim, dim) + float* w2; // (layer, dim, hidden_dim) + float* w3; // (layer, hidden_dim, dim) + // final rmsnorm + float* rms_final_weight; // (dim,) + // (optional) classifier weights for the logits, on the last layer + float* wcls; +} TransformerWeights; + +typedef struct { + // current wave of activations + float *x; // activation at current time stamp (dim,) + float *xb; // same, but inside a residual branch (dim,) + float *xb2; // an additional buffer just for convenience (dim,) + float *hb; // buffer for hidden dimension in the ffn (hidden_dim,) + float *hb2; // buffer for hidden dimension in the ffn (hidden_dim,) + float *q; // query (dim,) + float *k; // key (dim,) + float *v; // value (dim,) + float *att; // buffer for scores/attention values (n_heads, seq_len) + float *logits; // output logits + // kv cache + float* key_cache; // (layer, seq_len, dim) + float* value_cache; // (layer, seq_len, dim) +} RunState; + +typedef struct { + Config config; // the hyperparameters of the architecture (the blueprint) + TransformerWeights weights; // the weights of the model + RunState state; // buffers for the "wave" of activations in the forward pass + // some more state needed to properly clean up the memory mapping (sigh) + int fd; // file descriptor for memory mapping + float* data; // memory mapped data pointer + ssize_t file_size; // size of the checkpoint file in bytes +} Transformer; + +void malloc_run_state(RunState* s, Config* p) { + // we calloc instead of malloc to keep valgrind happy + int kv_dim = (p->dim * p->n_kv_heads) / p->n_heads; + s->x = calloc(p->dim, sizeof(float)); + s->xb = calloc(p->dim, sizeof(float)); + s->xb2 = calloc(p->dim, sizeof(float)); + s->hb = calloc(p->hidden_dim, sizeof(float)); + s->hb2 = calloc(p->hidden_dim, sizeof(float)); + s->q = calloc(p->dim, sizeof(float)); + s->key_cache = calloc(p->n_layers * p->seq_len * kv_dim, sizeof(float)); + s->value_cache = calloc(p->n_layers * p->seq_len * kv_dim, sizeof(float)); + s->att = calloc(p->n_heads * p->seq_len, sizeof(float)); + s->logits = calloc(p->vocab_size, sizeof(float)); + // ensure all mallocs went fine + if (!s->x || !s->xb || !s->xb2 || !s->hb || !s->hb2 || !s->q + || !s->key_cache || !s->value_cache || !s->att || !s->logits) { + fprintf(stderr, "malloc failed!\n"); + exit(EXIT_FAILURE); + } +} + +void free_run_state(RunState* s) { + free(s->x); + free(s->xb); + free(s->xb2); + free(s->hb); + free(s->hb2); + free(s->q); + free(s->att); + free(s->logits); + free(s->key_cache); + free(s->value_cache); +} + +void memory_map_weights(TransformerWeights *w, Config* p, float* ptr, int shared_weights) { + int head_size = p->dim / p->n_heads; + // make sure the multiplications below are done in 64bit to fit the parameter counts of 13B+ models + unsigned long long n_layers = p->n_layers; + w->token_embedding_table = ptr; + ptr += p->vocab_size * p->dim; + w->rms_att_weight = ptr; + ptr += n_layers * p->dim; + w->wq = ptr; + ptr += n_layers * p->dim * (p->n_heads * head_size); + w->wk = ptr; + ptr += n_layers * p->dim * (p->n_kv_heads * head_size); + w->wv = ptr; + ptr += n_layers * p->dim * (p->n_kv_heads * head_size); + w->wo = ptr; + ptr += n_layers * (p->n_heads * head_size) * p->dim; + w->rms_ffn_weight = ptr; + ptr += n_layers * p->dim; + w->w1 = ptr; + ptr += n_layers * p->dim * p->hidden_dim; + w->w2 = ptr; + ptr += n_layers * p->hidden_dim * p->dim; + w->w3 = ptr; + ptr += n_layers * p->dim * p->hidden_dim; + w->rms_final_weight = ptr; + ptr += p->dim; + ptr += p->seq_len * head_size / 2; // skip what used to be freq_cis_real (for RoPE) + ptr += p->seq_len * head_size / 2; // skip what used to be freq_cis_imag (for RoPE) + w->wcls = shared_weights ? w->token_embedding_table : ptr; +} + +void read_checkpoint(char* checkpoint, Config* config, TransformerWeights* weights, + int* fd, float** data, ssize_t* file_size) { + FILE *file = fopen(checkpoint, "rb"); + if (!file) { fprintf(stderr, "Couldn't open file %s\n", checkpoint); exit(EXIT_FAILURE); } + // read in the config header + if (fread(config, sizeof(Config), 1, file) != 1) { exit(EXIT_FAILURE); } + // negative vocab size is hacky way of signaling unshared weights. bit yikes. + int shared_weights = config->vocab_size > 0 ? 1 : 0; + config->vocab_size = abs(config->vocab_size); + // figure out the file size + fseek(file, 0, SEEK_END); // move file pointer to end of file + *file_size = ftell(file); // get the file size, in bytes + fclose(file); + // memory map the Transformer weights into the data pointer + *fd = open(checkpoint, O_RDONLY); // open in read only mode + if (*fd == -1) { fprintf(stderr, "open failed!\n"); exit(EXIT_FAILURE); } + *data = mmap(NULL, *file_size, PROT_READ, MAP_PRIVATE, *fd, 0); + if (*data == MAP_FAILED) { fprintf(stderr, "mmap failed!\n"); exit(EXIT_FAILURE); } + float* weights_ptr = *data + sizeof(Config)/sizeof(float); + memory_map_weights(weights, config, weights_ptr, shared_weights); +} + +void build_transformer(Transformer *t, char* checkpoint_path) { + // read in the Config and the Weights from the checkpoint + read_checkpoint(checkpoint_path, &t->config, &t->weights, &t->fd, &t->data, &t->file_size); + // allocate the RunState buffers + malloc_run_state(&t->state, &t->config); +} + +void free_transformer(Transformer* t) { + // close the memory mapping + if (t->data != MAP_FAILED) { munmap(t->data, t->file_size); } + if (t->fd != -1) { close(t->fd); } + // free the RunState buffers + free_run_state(&t->state); +} + +// ---------------------------------------------------------------------------- +// neural net blocks; the dynamics of the Transformer + +void rmsnorm(float* o, float* x, float* weight, int size) { + // calculate sum of squares + float ss = 0.0f; + for (int j = 0; j < size; j++) { + ss += x[j] * x[j]; + } + ss /= size; + ss += 1e-5f; + ss = 1.0f / sqrtf(ss); + // normalize and scale + for (int j = 0; j < size; j++) { + o[j] = weight[j] * (ss * x[j]); + } +} + +void softmax(float* x, int size) { + // find max value (for numerical stability) + float max_val = x[0]; + for (int i = 1; i < size; i++) { + if (x[i] > max_val) { + max_val = x[i]; + } + } + // exp and sum + float sum = 0.0f; + for (int i = 0; i < size; i++) { + x[i] = expf(x[i] - max_val); + sum += x[i]; + } + // normalize + for (int i = 0; i < size; i++) { + x[i] /= sum; + } +} + +void matmul(float* xout, float* x, float* w, int n, int d) { + // W (d,n) @ x (n,) -> xout (d,) + // by far the most amount of time is spent inside this little function + int i; + #pragma omp parallel for private(i) + for (i = 0; i < d; i++) { + float val = 0.0f; + for (int j = 0; j < n; j++) { + val += w[i * n + j] * x[j]; + } + xout[i] = val; + } +} + +float* forward(Transformer* transformer, int token, int pos) { + + // a few convenience variables + Config* p = &transformer->config; + TransformerWeights* w = &transformer->weights; + RunState* s = &transformer->state; + float *x = s->x; + int dim = p->dim; + int kv_dim = (p->dim * p->n_kv_heads) / p->n_heads; + int kv_mul = p->n_heads / p->n_kv_heads; // integer multiplier of the kv sharing in multiquery + int hidden_dim = p->hidden_dim; + int head_size = dim / p->n_heads; + + // copy the token embedding into x + float* content_row = w->token_embedding_table + token * dim; + memcpy(x, content_row, dim*sizeof(*x)); + + // forward all the layers + for(unsigned long long l = 0; l < p->n_layers; l++) { + + // attention rmsnorm + rmsnorm(s->xb, x, w->rms_att_weight + l*dim, dim); + + // key and value point to the kv cache + int loff = l * p->seq_len * kv_dim; // kv cache layer offset for convenience + s->k = s->key_cache + loff + pos * kv_dim; + s->v = s->value_cache + loff + pos * kv_dim; + + // qkv matmuls for this position + matmul(s->q, s->xb, w->wq + l*dim*dim, dim, dim); + matmul(s->k, s->xb, w->wk + l*dim*kv_dim, dim, kv_dim); + matmul(s->v, s->xb, w->wv + l*dim*kv_dim, dim, kv_dim); + + // RoPE relative positional encoding: complex-valued rotate q and k in each head + for (int i = 0; i < dim; i+=2) { + int head_dim = i % head_size; + float freq = 1.0f / powf(10000.0f, head_dim / (float)head_size); + float val = pos * freq; + float fcr = cosf(val); + float fci = sinf(val); + int rotn = i < kv_dim ? 2 : 1; // how many vectors? 2 = q & k, 1 = q only + for (int v = 0; v < rotn; v++) { + float* vec = v == 0 ? s->q : s->k; // the vector to rotate (query or key) + float v0 = vec[i]; + float v1 = vec[i+1]; + vec[i] = v0 * fcr - v1 * fci; + vec[i+1] = v0 * fci + v1 * fcr; + } + } + + // multihead attention. iterate over all heads + int h; + #pragma omp parallel for private(h) + for (h = 0; h < p->n_heads; h++) { + // get the query vector for this head + float* q = s->q + h * head_size; + // attention scores for this head + float* att = s->att + h * p->seq_len; + // iterate over all timesteps, including the current one + for (int t = 0; t <= pos; t++) { + // get the key vector for this head and at this timestep + float* k = s->key_cache + loff + t * kv_dim + (h / kv_mul) * head_size; + // calculate the attention score as the dot product of q and k + float score = 0.0f; + for (int i = 0; i < head_size; i++) { + score += q[i] * k[i]; + } + score /= sqrtf(head_size); + // save the score to the attention buffer + att[t] = score; + } + + // softmax the scores to get attention weights, from 0..pos inclusively + softmax(att, pos + 1); + + // weighted sum of the values, store back into xb + float* xb = s->xb + h * head_size; + memset(xb, 0, head_size * sizeof(float)); + for (int t = 0; t <= pos; t++) { + // get the value vector for this head and at this timestep + float* v = s->value_cache + loff + t * kv_dim + (h / kv_mul) * head_size; + // get the attention weight for this timestep + float a = att[t]; + // accumulate the weighted value into xb + for (int i = 0; i < head_size; i++) { + xb[i] += a * v[i]; + } + } + } + + // final matmul to get the output of the attention + matmul(s->xb2, s->xb, w->wo + l*dim*dim, dim, dim); + + // residual connection back into x + for (int i = 0; i < dim; i++) { + x[i] += s->xb2[i]; + } + + // ffn rmsnorm + rmsnorm(s->xb, x, w->rms_ffn_weight + l*dim, dim); + + // Now for FFN in PyTorch we have: self.w2(F.silu(self.w1(x)) * self.w3(x)) + // first calculate self.w1(x) and self.w3(x) + matmul(s->hb, s->xb, w->w1 + l*dim*hidden_dim, dim, hidden_dim); + matmul(s->hb2, s->xb, w->w3 + l*dim*hidden_dim, dim, hidden_dim); + + // SwiGLU non-linearity + for (int i = 0; i < hidden_dim; i++) { + float val = s->hb[i]; + // silu(x)=x*σ(x), where σ(x) is the logistic sigmoid + val *= (1.0f / (1.0f + expf(-val))); + // elementwise multiply with w3(x) + val *= s->hb2[i]; + s->hb[i] = val; + } + + // final matmul to get the output of the ffn + matmul(s->xb, s->hb, w->w2 + l*dim*hidden_dim, hidden_dim, dim); + + // residual connection + for (int i = 0; i < dim; i++) { + x[i] += s->xb[i]; + } + } + + // final rmsnorm + rmsnorm(x, x, w->rms_final_weight, dim); + + // classifier into logits + matmul(s->logits, x, w->wcls, p->dim, p->vocab_size); + return s->logits; +} + +// ---------------------------------------------------------------------------- +// The Byte Pair Encoding (BPE) Tokenizer that translates strings <-> tokens + +typedef struct { + char *str; + int id; +} TokenIndex; + +typedef struct { + char** vocab; + float* vocab_scores; + TokenIndex *sorted_vocab; + int vocab_size; + unsigned int max_token_length; + unsigned char byte_pieces[512]; // stores all single-byte strings +} Tokenizer; + +int compare_tokens(const void *a, const void *b) { + return strcmp(((TokenIndex*)a)->str, ((TokenIndex*)b)->str); +} + +void build_tokenizer(Tokenizer* t, char* tokenizer_path, int vocab_size) { + // i should have written the vocab_size into the tokenizer file... sigh + t->vocab_size = vocab_size; + // malloc space to hold the scores and the strings + t->vocab = (char**)malloc(vocab_size * sizeof(char*)); + t->vocab_scores = (float*)malloc(vocab_size * sizeof(float)); + t->sorted_vocab = NULL; // initialized lazily + for (int i = 0; i < 256; i++) { + t->byte_pieces[i * 2] = (unsigned char)i; + t->byte_pieces[i * 2 + 1] = '\0'; + } + // read in the file + FILE *file = fopen(tokenizer_path, "rb"); + if (!file) { fprintf(stderr, "couldn't load %s\n", tokenizer_path); exit(EXIT_FAILURE); } + if (fread(&t->max_token_length, sizeof(int), 1, file) != 1) { fprintf(stderr, "failed read\n"); exit(EXIT_FAILURE); } + int len; + for (int i = 0; i < vocab_size; i++) { + if (fread(t->vocab_scores + i, sizeof(float), 1, file) != 1) { fprintf(stderr, "failed read\n"); exit(EXIT_FAILURE);} + if (fread(&len, sizeof(int), 1, file) != 1) { fprintf(stderr, "failed read\n"); exit(EXIT_FAILURE); } + t->vocab[i] = (char *)malloc(len + 1); + if (fread(t->vocab[i], len, 1, file) != 1) { fprintf(stderr, "failed read\n"); exit(EXIT_FAILURE); } + t->vocab[i][len] = '\0'; // add the string terminating token + } + fclose(file); +} + +void free_tokenizer(Tokenizer* t) { + for (int i = 0; i < t->vocab_size; i++) { free(t->vocab[i]); } + free(t->vocab); + free(t->vocab_scores); + free(t->sorted_vocab); +} + +char* decode(Tokenizer* t, int prev_token, int token) { + char *piece = t->vocab[token]; + // following BOS (1) token, sentencepiece decoder strips any leading whitespace (see PR #89) + if (prev_token == 1 && piece[0] == ' ') { piece++; } + // careful, some tokens designate raw bytes, and look like e.g. '<0x01>' + // parse this and convert and return the actual byte + unsigned char byte_val; + if (sscanf(piece, "<0x%02hhX>", &byte_val) == 1) { + piece = (char*)t->byte_pieces + byte_val * 2; + } + return piece; +} + +void safe_printf(char *piece) { + // piece might be a raw byte token, and we only want to print printable chars or whitespace + // because some of the other bytes can be various control codes, backspace, etc. + if (piece == NULL) { return; } + if (piece[0] == '\0') { return; } + if (piece[1] == '\0') { + unsigned char byte_val = piece[0]; + if (!(isprint(byte_val) || isspace(byte_val))) { + return; // bad byte, don't print it + } + } + printf("%s", piece); +} + +int str_lookup(char *str, TokenIndex *sorted_vocab, int vocab_size) { + // efficiently find the perfect match for str in vocab, return its index or -1 if not found + TokenIndex tok = { .str = str }; // acts as the key to search for + TokenIndex *res = bsearch(&tok, sorted_vocab, vocab_size, sizeof(TokenIndex), compare_tokens); + return res != NULL ? res->id : -1; +} + +void encode(Tokenizer* t, char *text, int8_t bos, int8_t eos, int *tokens, int *n_tokens) { + // encode the string text (input) into an upper-bound preallocated tokens[] array + // bos != 0 means prepend the BOS token (=1), eos != 0 means append the EOS token (=2) + if (text == NULL) { fprintf(stderr, "cannot encode NULL text\n"); exit(EXIT_FAILURE); } + + if (t->sorted_vocab == NULL) { + // lazily malloc and sort the vocabulary + t->sorted_vocab = malloc(t->vocab_size * sizeof(TokenIndex)); + for (int i = 0; i < t->vocab_size; i++) { + t->sorted_vocab[i].str = t->vocab[i]; + t->sorted_vocab[i].id = i; + } + qsort(t->sorted_vocab, t->vocab_size, sizeof(TokenIndex), compare_tokens); + } + + // create a temporary buffer that will store merge candidates of always two consecutive tokens + // *2 for concat, +1 for null terminator +2 for UTF8 (in case max_token_length is 1) + char* str_buffer = malloc((t->max_token_length*2 +1 +2) * sizeof(char)); + size_t str_len = 0; + + // start at 0 tokens + *n_tokens = 0; + + // add optional BOS (=1) token, if desired + if (bos) tokens[(*n_tokens)++] = 1; + + // add_dummy_prefix is true by default + // so prepend a dummy prefix token to the input string, but only if text != "" + // TODO: pretty sure this isn't correct in the general case but I don't have the + // energy to read more of the sentencepiece code to figure out what it's doing + if (text[0] != '\0') { + int dummy_prefix = str_lookup(" ", t->sorted_vocab, t->vocab_size); + tokens[(*n_tokens)++] = dummy_prefix; + } + + // Okay UTF-8 time. This will get messy. Here is the reference from Wikipedia: + // Code point ↔ UTF-8 conversion + // First code point Last code point Byte 1 Byte 2 Byte 3 Byte 4 + // U+0000 U+007F 0xxxxxxx + // U+0080 U+07FF 110xxxxx 10xxxxxx + // U+0800 U+FFFF 1110xxxx 10xxxxxx 10xxxxxx + // U+10000 U+10FFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + + // process the raw (UTF-8) byte sequence of the input string + for (char *c = text; *c != '\0'; c++) { + + // reset buffer if the current byte is ASCII or a leading byte + // 0xC0 is 11000000, so (*c & 0xC0) keeps the first 2 bits and zeros the rest + // 0x80 is 10000000 + // in UTF-8, all continuation bytes start with "10" in first two bits + // so in English this is: "if this byte is not a continuation byte" + if ((*c & 0xC0) != 0x80) { + // this byte must be either a leading byte (11...) or an ASCII char (0x...) + // => reset our location, as we're starting a new UTF-8 codepoint + str_len = 0; + } + + // append the current byte to the buffer + str_buffer[str_len++] = *c; // ++ is post-increment, incremented after this line + str_buffer[str_len] = '\0'; + + // while the next character is a continuation byte, continue appending + // but if there are too many of them, just stop to avoid overruning str_buffer size. + if ((*(c+1) & 0xC0) == 0x80 && str_len < 4) { + continue; + } + + // ok c+1 is not a continuation byte, so we've read in a full codepoint + int id = str_lookup(str_buffer, t->sorted_vocab, t->vocab_size); + + if (id != -1) { + // we found this codepoint in vocab, add it as a token + tokens[(*n_tokens)++] = id; + } else { + // byte_fallback encoding: just encode each byte as a token + // +3 is here because the first 3 vocab elements are , , + // so the individual bytes only start at index 3 + for (int i=0; i < str_len; i++) { + tokens[(*n_tokens)++] = (unsigned char)str_buffer[i] + 3; + } + } + str_len = 0; // protect against a sequence of stray UTF8 continuation bytes + } + + // merge the best consecutive pair each iteration, according the scores in vocab_scores + while (1) { + float best_score = -1e10; + int best_id = -1; + int best_idx = -1; + + for (int i=0; i < (*n_tokens-1); i++) { + // check if we can merge the pair (tokens[i], tokens[i+1]) + sprintf(str_buffer, "%s%s", t->vocab[tokens[i]], t->vocab[tokens[i+1]]); + int id = str_lookup(str_buffer, t->sorted_vocab, t->vocab_size); + if (id != -1 && t->vocab_scores[id] > best_score) { + // this merge pair exists in vocab! record its score and position + best_score = t->vocab_scores[id]; + best_id = id; + best_idx = i; + } + } + + if (best_idx == -1) { + break; // we couldn't find any more pairs to merge, so we're done + } + + // merge the consecutive pair (best_idx, best_idx+1) into new token best_id + tokens[best_idx] = best_id; + // delete token at position best_idx+1, shift the entire sequence back 1 + for (int i = best_idx+1; i < (*n_tokens-1); i++) { + tokens[i] = tokens[i+1]; + } + (*n_tokens)--; // token length decreased + } + + // add optional EOS (=2) token, if desired + if (eos) tokens[(*n_tokens)++] = 2; + + free(str_buffer); +} + +// ---------------------------------------------------------------------------- +// The Sampler, which takes logits and returns a sampled token +// sampling can be done in a few ways: greedy argmax, sampling, top-p sampling + +typedef struct { + float prob; + int index; +} ProbIndex; // struct used when sorting probabilities during top-p sampling + +typedef struct { + int vocab_size; + ProbIndex* probindex; // buffer used in top-p sampling + float temperature; + float topp; + unsigned long long rng_state; +} Sampler; + +int sample_argmax(float* probabilities, int n) { + // return the index that has the highest probability + int max_i = 0; + float max_p = probabilities[0]; + for (int i = 1; i < n; i++) { + if (probabilities[i] > max_p) { + max_i = i; + max_p = probabilities[i]; + } + } + return max_i; +} + +int sample_mult(float* probabilities, int n, float coin) { + // sample index from probabilities (they must sum to 1!) + // coin is a random number in [0, 1), usually from random_f32() + float cdf = 0.0f; + for (int i = 0; i < n; i++) { + cdf += probabilities[i]; + if (coin < cdf) { + return i; + } + } + return n - 1; // in case of rounding errors +} + +int compare(const void* a, const void* b) { + ProbIndex* a_ = (ProbIndex*) a; + ProbIndex* b_ = (ProbIndex*) b; + if (a_->prob > b_->prob) return -1; + if (a_->prob < b_->prob) return 1; + return 0; +} + +int sample_topp(float* probabilities, int n, float topp, ProbIndex* probindex, float coin) { + // top-p sampling (or "nucleus sampling") samples from the smallest set of + // tokens that exceed probability topp. This way we never sample tokens that + // have very low probabilities and are less likely to go "off the rails". + // coin is a random number in [0, 1), usually from random_f32() + + int n0 = 0; + // quicksort indices in descending order of probabilities + // values smaller than (1 - topp) / (n - 1) cannot be part of the result + // so for efficiency we crop these out as candidates before sorting + const float cutoff = (1.0f - topp) / (n - 1); + for (int i = 0; i < n; i++) { + if (probabilities[i] >= cutoff) { + probindex[n0].index = i; + probindex[n0].prob = probabilities[i]; + n0++; + } + } + qsort(probindex, n0, sizeof(ProbIndex), compare); + + // truncate the list where cumulative probability exceeds topp + float cumulative_prob = 0.0f; + int last_idx = n0 - 1; // in case of rounding errors consider all elements + for (int i = 0; i < n0; i++) { + cumulative_prob += probindex[i].prob; + if (cumulative_prob > topp) { + last_idx = i; + break; // we've exceeded topp by including last_idx + } + } + + // sample from the truncated list + float r = coin * cumulative_prob; + float cdf = 0.0f; + for (int i = 0; i <= last_idx; i++) { + cdf += probindex[i].prob; + if (r < cdf) { + return probindex[i].index; + } + } + return probindex[last_idx].index; // in case of rounding errors +} + +void build_sampler(Sampler* sampler, int vocab_size, float temperature, float topp, unsigned long long rng_seed) { + sampler->vocab_size = vocab_size; + sampler->temperature = temperature; + sampler->topp = topp; + sampler->rng_state = rng_seed; + // buffer only used with nucleus sampling; may not need but it's ~small + sampler->probindex = malloc(sampler->vocab_size * sizeof(ProbIndex)); +} + +void free_sampler(Sampler* sampler) { + free(sampler->probindex); +} + +unsigned int random_u32(unsigned long long *state) { + // xorshift rng: https://en.wikipedia.org/wiki/Xorshift#xorshift.2A + *state ^= *state >> 12; + *state ^= *state << 25; + *state ^= *state >> 27; + return (*state * 0x2545F4914F6CDD1Dull) >> 32; +} +float random_f32(unsigned long long *state) { // random float32 in [0,1) + return (random_u32(state) >> 8) / 16777216.0f; +} + +int sample(Sampler* sampler, float* logits) { + // sample the token given the logits and some hyperparameters + int next; + if (sampler->temperature == 0.0f) { + // greedy argmax sampling: take the token with the highest probability + next = sample_argmax(logits, sampler->vocab_size); + } else { + // apply the temperature to the logits + for (int q=0; qvocab_size; q++) { logits[q] /= sampler->temperature; } + // apply softmax to the logits to get the probabilities for next token + softmax(logits, sampler->vocab_size); + // flip a (float) coin (this is our source of entropy for sampling) + float coin = random_f32(&sampler->rng_state); + // we sample from this distribution to get the next token + if (sampler->topp <= 0 || sampler->topp >= 1) { + // simply sample from the predicted probability distribution + next = sample_mult(logits, sampler->vocab_size, coin); + } else { + // top-p (nucleus) sampling, clamping the least likely tokens to zero + next = sample_topp(logits, sampler->vocab_size, sampler->topp, sampler->probindex, coin); + } + } + return next; +} + +// ---------------------------------------------------------------------------- +// utilities: time + +long time_in_ms() { + // return time in milliseconds, for benchmarking the model speed + struct timespec time; + clock_gettime(CLOCK_REALTIME, &time); + return time.tv_sec * 1000 + time.tv_nsec / 1000000; +} + +// ---------------------------------------------------------------------------- +// generation loop + +void generate(Transformer *transformer, Tokenizer *tokenizer, Sampler *sampler, char *prompt, int steps) { + char *empty_prompt = ""; + if (prompt == NULL) { prompt = empty_prompt; } + + // encode the (string) prompt into tokens sequence + int num_prompt_tokens = 0; + int* prompt_tokens = (int*)malloc((strlen(prompt)+3) * sizeof(int)); // +3 for '\0', ?BOS, ?EOS + encode(tokenizer, prompt, 1, 0, prompt_tokens, &num_prompt_tokens); + if (num_prompt_tokens < 1) { + fprintf(stderr, "something is wrong, expected at least 1 prompt token\n"); + exit(EXIT_FAILURE); + } + + // start the main loop + long start = 0; // used to time our code, only initialized after first iteration + int next; // will store the next token in the sequence + int token = prompt_tokens[0]; // kick off with the first token in the prompt + int pos = 0; // position in the sequence + while (pos < steps) { + + // forward the transformer to get logits for the next token + float* logits = forward(transformer, token, pos); + + // advance the state machine + if (pos < num_prompt_tokens - 1) { + // if we are still processing the input prompt, force the next prompt token + next = prompt_tokens[pos + 1]; + } else { + // otherwise sample the next token from the logits + next = sample(sampler, logits); + } + pos++; + + // data-dependent terminating condition: the BOS (=1) token delimits sequences + if (next == 1) { break; } + + // print the token as string, decode it with the Tokenizer object + char* piece = decode(tokenizer, token, next); + safe_printf(piece); // same as printf("%s", piece), but skips "unsafe" bytes + fflush(stdout); + token = next; + + // init the timer here because the first iteration can be slower + if (start == 0) { start = time_in_ms(); } + } + printf("\n"); + + // report achieved tok/s (pos-1 because the timer starts after first iteration) + if (pos > 1) { + long end = time_in_ms(); + fprintf(stderr, "achieved tok/s: %f\n", (pos-1) / (double)(end-start)*1000); + } + + free(prompt_tokens); +} + +void read_stdin(const char* guide, char* buffer, size_t bufsize) { + // read a line from stdin, up to but not including \n + printf("%s", guide); + if (fgets(buffer, bufsize, stdin) != NULL) { + size_t len = strlen(buffer); + if (len > 0 && buffer[len - 1] == '\n') { + buffer[len - 1] = '\0'; // strip newline + } + } +} + +// ---------------------------------------------------------------------------- +// chat loop +// I manually inspected the tokens for a few chat conversations compared to +// python reference and that seemed ok, but this was not thoroughly tested and +// is not safely implemented, it's more a proof of concept atm. + +void chat(Transformer *transformer, Tokenizer *tokenizer, Sampler *sampler, + char *cli_user_prompt, char *cli_system_prompt, int steps) { + + // buffers for reading the system prompt and user prompt from stdin + // you'll notice they are soomewhat haphazardly and unsafely set atm + char system_prompt[512]; + char user_prompt[512]; + char rendered_prompt[1152]; + int num_prompt_tokens = 0; + int* prompt_tokens = (int*)malloc(1152 * sizeof(int)); + int user_idx; + + // start the main loop + int8_t user_turn = 1; // user starts + int next; // will store the next token in the sequence + int token; // stores the current token to feed into the transformer + int prev_token; + int pos = 0; // position in the sequence + while (pos < steps) { + + // when it is the user's turn to contribute tokens to the dialog... + if (user_turn) { + // get the (optional) system prompt at position 0 + if (pos == 0) { + // at position 0, the user can also contribute a system prompt + if (cli_system_prompt == NULL) { + // system prompt was not passed in, attempt to get it from stdin + read_stdin("Enter system prompt (optional): ", system_prompt, sizeof(system_prompt)); + } else { + // system prompt was passed in, use it + strcpy(system_prompt, cli_system_prompt); + } + } + // get the user prompt + if (pos == 0 && cli_user_prompt != NULL) { + // user prompt for position 0 was passed in, use it + strcpy(user_prompt, cli_user_prompt); + } else { + // otherwise get user prompt from stdin + read_stdin("User: ", user_prompt, sizeof(user_prompt)); + } + // render user/system prompts into the Llama 2 Chat schema + if (pos == 0 && system_prompt[0] != '\0') { + char system_template[] = "[INST] <>\n%s\n<>\n\n%s [/INST]"; + sprintf(rendered_prompt, system_template, system_prompt, user_prompt); + } else { + char user_template[] = "[INST] %s [/INST]"; + sprintf(rendered_prompt, user_template, user_prompt); + } + // encode the rendered prompt into tokens + encode(tokenizer, rendered_prompt, 1, 0, prompt_tokens, &num_prompt_tokens); + user_idx = 0; // reset the user index + user_turn = 0; + printf("Assistant: "); + } + + // determine the token to pass into the transformer next + if (user_idx < num_prompt_tokens) { + // if we are still processing the input prompt, force the next prompt token + token = prompt_tokens[user_idx++]; + } else { + // otherwise use the next token sampled from previous turn + token = next; + } + // EOS (=2) token ends the Assistant turn + if (token == 2) { user_turn = 1; } + + // forward the transformer to get logits for the next token + float* logits = forward(transformer, token, pos); + next = sample(sampler, logits); + pos++; + + if (user_idx >= num_prompt_tokens && next != 2) { + // the Assistant is responding, so print its output + char* piece = decode(tokenizer, token, next); + safe_printf(piece); // same as printf("%s", piece), but skips "unsafe" bytes + fflush(stdout); + } + if (next == 2) { printf("\n"); } + } + printf("\n"); + free(prompt_tokens); +} + + +// ---------------------------------------------------------------------------- +// CLI, include only if not testing +#ifndef TESTING + +void error_usage() { + fprintf(stderr, "Usage: run [options]\n"); + fprintf(stderr, "Example: run model.bin -n 256 -i \"Once upon a time\"\n"); + fprintf(stderr, "Options:\n"); + fprintf(stderr, " -t temperature in [0,inf], default 1.0\n"); + fprintf(stderr, " -p p value in top-p (nucleus) sampling in [0,1] default 0.9\n"); + fprintf(stderr, " -s random seed, default time(NULL)\n"); + fprintf(stderr, " -n number of steps to run for, default 256. 0 = max_seq_len\n"); + fprintf(stderr, " -i input prompt\n"); + fprintf(stderr, " -z optional path to custom tokenizer\n"); + fprintf(stderr, " -m mode: generate|chat, default: generate\n"); + fprintf(stderr, " -y (optional) system prompt in chat mode\n"); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) { + + // default parameters + char *checkpoint_path = NULL; // e.g. out/model.bin + char *tokenizer_path = "tokenizer.bin"; + float temperature = 1.0f; // 0.0 = greedy deterministic. 1.0 = original. don't set higher + float topp = 0.9f; // top-p in nucleus sampling. 1.0 = off. 0.9 works well, but slower + int steps = 256; // number of steps to run for + char *prompt = NULL; // prompt string + unsigned long long rng_seed = 0; // seed rng with time by default + char *mode = "generate"; // generate|chat + char *system_prompt = NULL; // the (optional) system prompt to use in chat mode + + // poor man's C argparse so we can override the defaults above from the command line + if (argc >= 2) { checkpoint_path = argv[1]; } else { error_usage(); } + for (int i = 2; i < argc; i+=2) { + // do some basic validation + if (i + 1 >= argc) { error_usage(); } // must have arg after flag + if (argv[i][0] != '-') { error_usage(); } // must start with dash + if (strlen(argv[i]) != 2) { error_usage(); } // must be -x (one dash, one letter) + // read in the args + if (argv[i][1] == 't') { temperature = atof(argv[i + 1]); } + else if (argv[i][1] == 'p') { topp = atof(argv[i + 1]); } + else if (argv[i][1] == 's') { rng_seed = atoi(argv[i + 1]); } + else if (argv[i][1] == 'n') { steps = atoi(argv[i + 1]); } + else if (argv[i][1] == 'i') { prompt = argv[i + 1]; } + else if (argv[i][1] == 'z') { tokenizer_path = argv[i + 1]; } + else if (argv[i][1] == 'm') { mode = argv[i + 1]; } + else if (argv[i][1] == 'y') { system_prompt = argv[i + 1]; } + else { error_usage(); } + } + + // parameter validation/overrides + if (rng_seed <= 0) rng_seed = (unsigned int)time(NULL); + if (temperature < 0.0) temperature = 0.0; + if (topp < 0.0 || 1.0 < topp) topp = 0.9; + if (steps < 0) steps = 0; + + // build the Transformer via the model .bin file + Transformer transformer; + build_transformer(&transformer, checkpoint_path); + if (steps == 0 || steps > transformer.config.seq_len) steps = transformer.config.seq_len; // override to ~max length + + // build the Tokenizer via the tokenizer .bin file + Tokenizer tokenizer; + build_tokenizer(&tokenizer, tokenizer_path, transformer.config.vocab_size); + + // build the Sampler + Sampler sampler; + build_sampler(&sampler, transformer.config.vocab_size, temperature, topp, rng_seed); + + // run! + if (strcmp(mode, "generate") == 0) { + generate(&transformer, &tokenizer, &sampler, prompt, steps); + } else if (strcmp(mode, "chat") == 0) { + chat(&transformer, &tokenizer, &sampler, prompt, system_prompt, steps); + } else { + fprintf(stderr, "unknown mode: %s\n", mode); + error_usage(); + } + + // memory and file handles cleanup + free_sampler(&sampler); + free_tokenizer(&tokenizer); + free_transformer(&transformer); + return 0; +} +#endif diff --git a/c/llama2/llgo.cfg b/c/llama2/llgo.cfg deleted file mode 100644 index 5100faf1..00000000 --- a/c/llama2/llgo.cfg +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cl": [ - "clang -emit-llvm -S -o llgo_autogen.ll -c llama2/run.c", - "rm llgo_autogen.lla; zip llgo_autogen.lla llgo_autogen.ll" - ] -} diff --git a/c/llama2/llgo_autogen.lla b/c/llama2/llgo_autogen.lla index fdb470e0d9c0243cf6248ac4709c37aba2d6c394..fa262434bf34936a4b6e80f21edc6a536149f594 100644 GIT binary patch literal 403 zcmWIWW@Zs#U|`^2(CWSsk+ZSwz1FDEB7gp+}} z-nufa5r|7GxEUB(zA`c}fDKqW@pRT<0|D3Xzq-0RJ-OA&_V{R&T=W(H%cdNi9BQzw zNABj7f7MTpEbRB?xtOM6QvA-*>e1m4|w3@2AB<;3W+s(z(->hs%S?vC%HQ>tZ zi3OY75@k)RtBy_2c-FnlibG+25L5I@q5TI$?s#2T^j(>GOJ3QT)l53kNl&iN_;ug1 zQhzpc*m`9(_fzWETZ5K5$gf#!X}rUu|MdT@`y#LPM=VqS`}DO&%s;Q1ZsP!NMkYCC rTp=X^3^N7>U{EkDX#}w_!i^OYZfIc^;LXYgQqBm3p+H&#WC8;Kg>98s literal 26554 zcmV)IK)kV?w_ChEr$=czr6o&ZS^m${%|ei@JsjqAMTX&|Ng!EdwK5Z z@%rH=ymtS1>n`pe+&@3u?l10-KMM7+``CSTcJ}7@^!R-7^zqZD<8OZZyI;GzpMH8c z?EZasJ-k2PT`M{NN7(5<;EB)ASM%lR?%iAW*E{fcx)gr3dj8ew^ecNW{SF!Fzs|$^ zR_%Ut5B|aZCFZ~QF}jcc2c7W0&NIG$*#GhQaE71Emjk@v|9QUq{nP&GkZ!uVx!*sz z`+lDP1)u(>`@5^_@4)ENKJotn&;HBf{_c2n|9E?N{QnNu-+g&H{wGWT3?h9U&o1p^ zKH_w3-h^qY^9S)>|I7XNhr8e2T^@d1-OzB`^`GI8++AQA`TNir`PK*-DG|(&v)IV3xiV?^VQ3n;|o`3t2e#y_ICep;)SN-g~Rf~i+S#j zYA(yuwiPc;3mONwdpztf3-9YI-Y1s#{X_TTn@)cD@~{7md)KgqUi=L6SBw?(V$z!J z@9yrOc89x*`^&@bfBvuE{ciWlT zbnRZ@{?1-iaSTkN+T;E0;pxlu-FLg| zgHupE8_icUFe#_)ynso46cxuTZ9-`$D~^`zmbdILzFZ&vyCr^D^M&~NV_p04lRH{bn|r@M+{+Am-DU&q5^_MT6l!mS@K zzU-eq#;<>J|Nn3k6{J(+AD6d(``iEd`+xbv?jQc)_y7Oz|C{3jzY4tSAO7L*=6{Eu zcHjP8xNmI~&u*L>#sHY*G=}15;-XdXBavwBSI6UZ_?0f*+28Me^L%&VH)!{(r>93p zL3i#<-dx`u+#k<(KV08kI*Ijkckg~U9v^nkKRoR}-yGf^f4aMPcXoNW@^}8}{`zwF zO_aNzoc-YF&5i3VRg>O_SGs3*Z@P1$aDi*$-R^I?-M^phc5gff2yfkIIu{^Sccad4 zoZgFWGwHUV;lxYYT{=G9KOXXrpph=Wx}Oz%FMfEQ&>N9W(*3B>fc){Agd!RBsVDuZ zafa66=5TwsdwOs>()HbEzl6u4D!UBKM*Xjw{>Q&=m7`>G z(OAPsf~2B@6&)IFkVCi6H@myz58=r-!?}MlTvIQfif)*SP-reHywGrS(RePNN{*M; ze~J?kUNskHp}Y8_ogX&%+5Y8o9a`qv=QJipZmt6wuk^HXbYg0n*60MAw;;kbzwa0B z{Cdl1_svB($#1=RvG5MrMSElZK|5X|!pEg2h|kwg7hVhS&->T8cSrLo+>wz5;58?C zz#CaX8p&W}IE_@-&sFm(2Hh0Q;;v{Pm1TtM^$~5NL6q zbxxdbv|{#IBlYQx)@YyIBWa(tIl0eTm-bm(y3g9+eb$+X+qTaYVs3Sxb)SiKywBRg zeID@FU#?b}7PU%``92#?uQFJ#G7{~x5n!Ko1 zv%#WHXZvh4*k@z$*I%xOnHD|FFnXBb^e}_ts599=M{av_hxz@7OclYtcOCZKP|-iq7dsdA=VQi*0%|#-E;UoORl7Ae-}D%P8UlVSswVuO77cvZuA2boj-pt%K@v@N!*6bS=t?`*NmYxgR-o#eE3- zGds+!{8Mn>PxCAHhT^`9jAQvX;(ZrwJWGgHrsYO}R&UJrT|llSgd`(qq}&=O_=y=X z-XxJ+;W%?AJV3}VxWun7ZXb4caN3Tn?w)UM+%2Eom_)eoe0Q_|e7Ny{_vvpd&(7Qr zzuay5zrWu-JRUB6&e9th$C0Ve8<(;|hd!~CSdoqa4%w|N<%ow6yz1FfdPh20N-5Iv zc~>-AN-1Bylx`p5NE+#~*#j?#3qJ0#T%k+ig%^^gOPSN91av7WUCIW!bUP78(ny)I znKI>&t0_XQP^Ki5saVQXIb|w9nTk@TYN1RSDKq!FZ(Gb7*ZcZn690RYc4~z(6`@SU zQl`o&Qvu3UlrmKdWvWP-s*y5PGi9oQQ9twwWhz3Mnx#ygQ>F%#sVQaZ7RpqSGIb+m z>SoH+m@+3#O0STnrZIJvG<8mz8jz-@q^Vm-Q%BO&jijlYNmCPX9_6`Np-oL_)3CH@ za@sV2HVvgs(?Xj%(xz#oP18)9Cby8>k1M2U2x%IYG)+#L29Tzqq-k16(?rrVjihOs zNz)9O+2Q85LYan8re!J9=9FmxWqQXpT1?wQnSN^`wMPyTxpEYAQFn;ySk!Q%$uOz< zc~~MPP10YS>A7UPlQqzbf$9rS@>hk@fR7(PzT16s_3(Io_jHxr1Z>$5s~FNRg$;hY ztPrz85`at}7%-qMyxr^>ybY?Wo z{uTUUjPb6@;)#4-Gb8jNStOHsK5Qidhi?8hQY@XCp#!Mt5@V$n=ejt$CMJ}4t2 zlx>_)wz5Lm=Ze29Gee2Fq2joqtYwEPvct`3rbLRqa++yS*HxJv(mYd+9jaP(s3JR5 zZR}9BvO{GU*MutbLxuUF;`pJe<%cTr!_8T!L>+qNEY#>w&}D{5^H4d4sB0Ob4h#`$ z_E=4(Pn5Onm)0a5Ztov|`gFU0`1C`p-dNIi$z>f0ruYaHE-CL)v3P5yLZSDe z(hEMjB{7lAwIwl;#8c&>lxB0q;|)3{C}`bM9$UVL>Q!ZI{ zvf&(7DV%FC7%VUN%T+VkTz!3|Y9@nK+3WRcrWcO4g;+_5H5EW*;wJRnclHtx-eEwPRyWrJBpqO}eLEbWc!I1nEo9dEAk5d~t7#;b~n- z7J5Y@5(=HGliXLcP0^}O>xf66dJ;wFRqg5atbwU zF@eC=JzS_2LUtt|pB!U53mTjARv$wYAp&{%8~d?@{v?0-DTz9sv=XEMQhF8ja7 z+5bIfu#t7h%0zmfaNo!3ESz}V!@TQ$gp1Vs(0QM#duq7 z4Yf!Mg~sZq;1bVEkg6wHQ+`w~iV^uS9yuS-NSt?ExfnjiV^%HZD~8;Y;UWfP7$G3T z*uzEgrBCkmaABxCJL3xbCdYW3$}rqlGs+#u;um7wPt90j5zy-%+9%nK6cK~X z79zdq$9p^vnSO;Q9Jcw*j5b+d-k|0Fyp=!uK9Is=( z;O_?|LjrMzP*mj~iPst&#El+e&PB}oh^doU;3{Sz6q>F<)AdB+ABop$4OSsm_&Yi0 zp#*38ST3VNhF$S1wQNV^SqJS$y(Sf5`%xFCP+>wv56Ds>r#R&@tQ>W@rM{U4Hd+dt zL)v6Uw~~`tq1Xn}ZE*Smx^+(hx($BaM7Q3N?jMQQ${}4sCu@lXlx{-~LHroQkma!Q zm2##5xl?5hk9N9M6`qjobgQt7B$wHtdJLARUGtq9#V!CTS#nkgj5g7XoC zD?uQx1j4uy3h7i+im|fsT0@MhrV3_dx?I(uX2V>HNL!7KW&aeIu9kSZF~@p(e>wYv z9E-K86nacU{Ym(kW|*li2N2e3ROp?rA3_lN=ZGPM5*d>CL{|hltV0q!8dXIcL2Lr8 zqH*Y!NP^&Cs|1T%B?xepAjnk$L083El|;9Ch;;(-T1(iyNpUus;Yx;&yc5Macy{4* zEt>9trfY)IYo@ZDYUyQH_L3e`UkoyRj3G+ft$d}NtVHk3)SmWZy5?Vg`7yOvO=3#; zE%aoCIuA4|U2#k|VNnsfZ33eb#KRovmW87d1dmEsg707vzJo=MM&ak~9|_m`mj{nT z-x57?UuL)x3q~b+96%C1jv)D`z;t_!r|W}A?vZ${T6*;>dr6O}clgGS^?6)CU<@eo zm1-rt&(xmwW7=1!L)(vOPt>G5i$(O%eic0_c^evNa?K@Pk;cS?G{wNu!=pQQ!NG#f{`fNi_1E>wT7UU zM8~xVo@HM-!ecO;x|LOO?K|OXbwkaoscb!^`VSrxCnK_;R@T{=vQC z2^V&psE=5iFDaUU6*Fm$se)uOZW4lYDy;wo5TR-w#-DdpB@|avE#lJ4N0awu3jUW}xq!pa5x_^MOmq)y=ZS47)J)`(k`u*4`9Yq}PaU?A z8EWh)5Gf@yap56(A82LzJA_e~w*@mfS~y$@aB{3n{XiMpWjDb$&>h-+^VPp~;?tGy z#`hJejJ4-=APuRRs6r48A1Bd_KpOChFT)5074+BO$0e#S<&JA;7TMDykmk$~h78hU zO!8a-oBL56{xQ~U< z@T5fb+Py^pwKts%A7Y#T6lZ6u&)%x7Vf+AX%-J2p|W6$HxCi z_)iPS1i#sTJ|e1e@9=aLD=}V92)cz3bWR9hC#D8B{yt0xIpu{mcoDq13BF2M(30dz~wbbiaATkr5!Ng~D z#fw1Y2soeyGn9l_clhJHt!mb-LJQoI1$;RqP?ONv|v@V`kjFqNWva z>#8u)bizo}SkqFB0=`|k+7^i(*Vl@Q^9ibFKzPvNj2v4-yP)_5ketQDYjJCoaD;ph zaeRclL4cXwn>h*!6e`38@ju^K$~40nv{`QzOM0tV5}A5`%1&JBU|_h*l475s`lYd` zh4-IX(nzHPr*|2K51rpVJ>7eH4KQ~fKr@1O>16jwf$S56Vz1B6`WC&;lIA)MC%b=D z8_E}&w4QAJDkr;dGMVD~ntFojR}yux%d)Re)a@s8Cr|CeF;4P<*9hj~sOhF|Liqfw z)Ekj;vu%Q~at_Ggr8qJ^fLwHx_DjMf=%zz7K{w74gif^w3LTONv9q9IvZgO$v*aJ8 z(^!V?ncU9)Bf;M(hrLIq`>;8jU9FaRxz z5KVazTAtwQKM)e?>%uF2e!jlB{PdJg0Cy^I*Z_M*S6jWqcVOG1==d+Xq2HI#iQd0w zo4Lp4FS7|j8Zt6Q#)-D>cq%o+sA=#Hp66e>-s-(!aNDwNN`vE)kk=C8Yey%xWx$t) zCvXalS|T{jUNXLIVb0?Ca9q=1k-8dRG;3nBGQK#{_@WR@fKezK zVHD;ajs~@?ht6>#|#ew*SC9J7o ztyn@*%o>)Et!0%Yw9JrZFdx6d!7@n*F)U#VA;zj$%Lm6_vW~AqiUf^Yo@KDXb;^Mp zO1_X)7QooR`E&XHyt=jgcze8aOY)Zbe7f8X^kR(RJm(9-2{Tt;14VdpB*HYbuKXZO zk|Wo!+Y#1;G04G-WP|SX@@+522?p=kbsxL%e}}IVA_qzQPVeg$7K>^Tf#eC@9$OB) z&wsMFGxEHm_v_~-pgeeHWybXrXq2vjf9{cvcAwp&Se@gSx4H^RB$;dL0BkI zKFFzxr{$ZQn?K!tyt;k9*; z!!%g6m{@0XGSumP4tY^mzH(*SCZGM}UQTyU+2R}O2r8sUUHKKC-{_8F_oMptuk~++ z|H}-K8F$%tOH>0ea96*pJC2Z#e|&s`RIFmwNyv!;t09n#aPOkS(_(|@5IhZO$8I+_ z5Jgc(UppIHvTBz9;Hp4`>u&C~X7GA*)W!v3u-LDj76AQQ&2 zLqplI24X@v zrgYLdm{~6>=2H^7G)7Egjg>BWIE# zW0McmG6v&HX%e+kF&&nlqZTT<_@Xv9WiykD1P9AuNgB@0T%PyTMwL%c9Lmim5q1MHYIA7bLSZV1F23mVy?{DKMbi zfe|g7NGNP=p4w^7#Y(z`U_f*M0nz0KLYVoGpf^d@KD%Pn)M? zBLDf;DR&it-zjVD83-4sbEBRMXMLDY%qGU4JdLOkbf95K*F!R z5!<9gTAl$H+`!F7v(JC`tgmD_W6z7_P#7`S2z4iz0ej=xJ(?J>1r=CB%4-?jT!tp+ zUKl{h*@=m^ueD-&j+RIZ5$r(ep#!C_ccAo3kB?wP^tv<=T!|EFD4<`u4;_U@di;7* zdPSnA-e(jw3G`HeabnE|dj5RD1v!5c6x|yAlk?{cpS~q55;pBW#RY?;;;Z{SZud{O z&o{3Xg(v^k$x-r3ad+lwZ3Ex>z&k>w$-OKPnx@*9vJ#6gcIhdH>>0Ef9I`J2Qr;|f z29CyU2(@@gCYs!_q4`_U)j?dcd}mz}*v>_ZFmvwD><g_C`vg>UK5(jw)uWlM==hCD6Qf@=k@whZWGFey$ZPl!rJm9wN|ixMtG!`jQ-dkqKvwz=B6X+QpS}YEefj(n5n-;57lh80i06YhFNoEOS>*wry>?PvPmS zvQ0u-sX2Iy(g3*PiA?qP3lv4q9`{~ss?UiiPNdF$y6=IX?c9x`u7NfUI2;JdFkd#;$6$kUZ=!P=jG=WD7L|tUj;M47FpiX{sl9xX>MOJW76hc zW;0GTH!;JG{`{Vn8k($H3kckb$1Ccz=BJtWEr4nW-@ufOWy9q6ECCZgP(X2Ltw8572Wx z)_bmpIg4=28;Xs%?;rMWlnV-p!V>yAQ3!}p0tkq)tR>N}8tz>hzN^9?Ibwh02>p>G z>PG2%B*mZ0X;4{j`;Hsq2q7=Jgk0?b;DU?q{`3DnTs$4G|9tr9e*E~*T@P;~@|O=~ zU5nk-{^ocHe>u86q@N-}P?xZThX8qbd$_%L`03Ndm+vtI#psVF`eAabrXL8lydSSe znoK#138nIVjP0fQ1Mg+$Gq<~~K5H2DkEMKZek=?J>4?$2T$d4DQ~RzCYcnz3yyMt_ z=t#zSN4iXQsLN!xWKKT&cweae?LxYF2XZGxmy#!ifZ3CS+k=*_OMVxF8#2>*H(6Ig zR4Bfy=9Lg47t=zNBWQxQR1qs|hMZxM5mK|55~6~@wzq{8hFg_#@u46bRe;*{^1rxD~Dp{7Fv8Ol-W21}jJ-cZC|c zE8NIE08NNk0w98a+D-KCV(Drh9#upm6)LHu1ZF~II_WmnrYaQBb=Iax*R^;mR9W8? ziu9ai)AX~6l1)@-6W4lp&_k$Ztx>4>eYRf*p&>qcPD4}kUZ+NtF^NXU{U*ux0%hNe z4e9qnEnUE4U{~nuU~Z!WU2!T&p>byD%8Yos7U{PLXXhuoa$>^%+>~9*HTFcE#!?JuO7? z%)W366~LG$rhq{)BYoR)W{;pjd%G6=d!ZU^x5WZ0)nJQ#c#)ygz8Gk#M6*hs;T6@b zugLuoJ>g9~;!Qo)Wo*!8`U$bI0{cYtq|bS;(CZC)z4zn#N8+{mBtT*3f?oH=_%N$r z(6OC%Y9TCMHu+sx$tc~{Fh9BB8$r%dWLNg; zxlfWIynaMUc0WsR^)By05PHU!r%GDB1AlUnqsXozup~PSDCQr6xE~!n^6smo3GfT$ zF>Xw$sv%KGIIwVB0B6#tg3HZVG6=0TXr9s4BzXW+QixOI1eOW1JJlxzU#|=W2up<$ z!0iUW-KvvrUYnu25$ts2f|8Ydw3dohxC!`7L_2*v+)lHJz%x>v;q*zl6!&G&s~lNo zHwAA1K*N+WwGNDTu%eI-cF11Slh*`T08p1Ky|33l87?apZSvv%2T+>>KqY4x8eSZ` zOsjPWN~tmoN~r({Wd;$%Z6pEI2~J=%>*xhevsR_Lm+W()Y#YG)N$)uJFUUs?Dz_!! zfU1Pjk=@vH>wR&5L=d&g=4iA_Rn+?(0DYg{TB^p%KF^k89k1{3=kuy73Cpe|&U%ro zH<%v28h5bNOu1w52g~RWmcbvqw8fUl9V~UUm6yiYa_+}0lePiRuuf;VkS4uwW?6s= zlG^yL^;^M#5N8hcjdGeFC$30zMrw{VP}t<)+@(tyj=t^zXV$4cSLzg1mgC^(wY7c< zwPcSP59{EiGYSq~)H+!DjNuLp)PQ~u22{Kn-??COnG_mmRCP?715cV(2c9&F(HKqH zfhP@e;K6PVonMr#5E9&?w1pU?X$>(*gF_6l4%n0<4AKm1i?SKl24S0p^JAj!8hp+V zU#@4i_-)v%c*D;3dky0Z|F*k7WFB#8VY-zhYW<4mIlGnBY6$nW1jF4T5O=!)#of-) z2RT1nlzo=rs~_f}U!%#|yd>N^JR`^LNlCaxn27y`qHsIAJls3u z!$n-gzKo$MdkJo&#qlKsMY@DV*cz%jklw#5pDu1Q(FTI+7^zf!I2HZW943G*-yOb7 zm!k^b*OrRQUL`fO2P*00`;{xLcq#qxy?AMB0b<_{$i7{3vu3+AgA2(agJ+gKRIvuK zPeXA-$<7bO4JG?D6gQMkzk~@uX=$K1+HwdCbBG%}RB!MAHgz!I?`ea_`VAgyHh9d6 z_~j^O20{F#68THzxWR)lF1tTcKuEtj@J?M&H%G5-jxFkDZ22u&cPv%Qf;*P1i_I$A z<=9uK$)vX3tC!Va=$HX~*Gqmt-NSPVg^4t{t*6ZXT*@ z(PsrNk6COORp<$#eCDod$=uaJ&+*=zur^$}1pc`D=gZULR8#J*>X=F-#pdoRlDn(A z+@?|?JJ&s*Oci$<@NfZ`Q|obXy9N?fF*{e~t%^5k<*KGj@v2s?1d}eIL|JK7LXaVF zS}P?I7QvhNz?8@mZ&c_TrNnd{C*Jt`A?Qzi;*Aos>Z?SatEEJb=1ZpA7Ccfl-EvRt z7!DpyU;r!li-HV+MXRDamiJJsKh`y^6U)b%bWDtmmDrYvis&s$<`fj#87etlCln+R z1<4H*BxemkNrr+Xq97Fb;@EisN=V7U!)-UnR^{XVo^{3fad7}emXJ;RCEQbq#64#L z8IpUI0*)S~Kzu+YM~@nu>Z3Nph@;K`6U0TEqerP$M~_k@Q!Ao~sj{a>DNV+}hP4P# zx59-_{vklOsve3-sTqsOy)!;sLlnGUn1YIk?C_~T6cI#NX>5G?`@tl1z44{goaCfp z&nCq-mZBU@vr$Gm)4(B?B1xr#QV>0$ignjgDW->P8G^S~>A2veCzL`>q11-1P)aX| zZ+gsmkU>337XaK@73>3LeBS}n5C~M>qvIJyTn{daL?P_h)L*(U!K+JD zzR(`ZJr0C_-A6jRkVp zh_+jNAj%r>cOEBHwl94V0;YQvVKdA(BFo6zEGq}dwgeX|HkN%NEN#ot*RTm}EKA?M zvTtD<)A8!r`~k9IJ&im2$50Q;;C4OTys?i3WUA1ML?fKIFYOdr12j#{dw~;LY%%MUjjN+-hKL zSUUB*CC$L}q#1ODUQdYM%a)-RE5?Y*Io2p9OQLRC$8#?iw05|e%`NAj{;J=IVg1^TZz)6_^j&rxKpRe9*yVN-!Ueho>C#LCuN6^DmRvA+-wVMD~j& zm=8#8FJbI<9;Uc!$TQF(V(xAGFhf>zCGD8lj8}qUm|zqZSA!0Dts258Xy}{;Fbdi! zvS+#lM!_$PA4emMf^LRU&=f{N#P^mnt~J3?(7Y3tY>L3##y#=BXBu zC`1iW@ENU=+>`4Nv-zon?`u9kmEw(SH9uC`X-<{5@uV6K%TUC&791eoEcqr_>QaL$e-Ay4=A-iJMD!-y`iV zVN?a5Ahx6Q^Jw84imr_DZ)~tewV5W(5h7*hCd*9GCeKXK34?Y;g(WNFJCK?d&eK`4 zGQJaqX$bpkGxgqJ1s&mIHp@)WHt$BEt>9mP(n83}c%E!q{3bDdK{rAr=w_${O`sAY zTU~)l&;%+$=THe7r%(ZlaAFD!jHzCMK+psNLFeGO&6Sk81~qqcC8Z{Cy3w@i%U&1V zcVt+J*%m7)bqySYCU6WI!7*qI$DjiogYKtmRh@H)eC?5Z?Fan&piaf?S>4x1Y;?b< z<^2SWUjcNX`Bd4BXb+kLcF=ZUew^_W(l0ZFE%-$gDk-%ruzT~C- zj%YNoI|tT+rps$`h1olb{o_Dk9#U{odr#@yThHh4xgQ#Y%rMQUt>QxutrZ`7=o#ib zl>r8O z>WjgQm|YV_OfNpIZ}JEPvEqbVZ_L=3dsP?nK=~~{6Z02+?XOqQEf)QQLRS5QP}98maJ^_8{*|Psx5`W z=++bpqnH#5qszv;(L^24S&2bAM_F^nznr7Y@DtNi$M~t09%0FSKMl)NaFm*syoNaL z4RYKYjT{Tc`{w;4@fw?RK{$%($m?O`)EI$Z@2vs%NW4~WXc;}*BqnC<89sxTOr1yX z-O5oEs|j#$${w`7H{`U&g!FO^bWo_*A6eCkTteMtc-lIaXHDtqH!f~NjfE)~g5H>851KkvY zQ(X8*A&H7$5`bEED z*7wA$XK{lm3HL<3?U8!B!IN$JInTyKPqrCKwFjohwFibE@s@Wbpg*9^9fG->=O3Fv zuDqt`6r?vcQbA3Fv~?;V-OwTLjhq8$rMH|Jv0Y-jOsQ$Mb!@k-qjR0DV>_(;OrI(%KeiKO zSV{RQ+B%knmLLkh?Qkxte<(2)+s)aJfn!*o6bw_~Y)T3NT?CyxZ8;v3?8usBE@B`=s|Zr?dR2i;%mh;qc$`N+l(e* z37SM;&UOV&!V)wI%c4opO_4Z<(mw=_+8SO2Jx&Yof_rCtxRgU_2|9#baRHM^m6fbj zO()fpNm(?h`W#BTcvx*nuXg`a%AvFbCCn;fDEVj4!?I8X%BrsT9&l!raI~!&5+iLy za<+<1j3jHQvcyQM^2A7+n!v5_8lph;ejW{pk@1Yel`;N}4W1jTrb$=C7p$5SBO_rb zoD>;<>-mRKFITcEgP%h~QsmTE)T*-bLl4tRLUwj*R%X+#Ch3aOo9$|nCgnI?{8qUf z$ENIpX2hj;B`x%H7X|_p7I;$Kx-{eA|2h72cd?AZjd_20cYk--9Ul%C&o}$0{pXv*?uYA} z8#kNh$GbGlQzgUa-q5(dz1=^4$N%XDX@2HGLVrAJ7%^?yUw8raUfHhn} z-4U0Scj$mP0qH*8?f%^#+@Ox%+_=@<1?q%}tHn!`PU;KbHHX~_G@}oE>E0uZwZy?RDKQ2D6#w*nL9{x&W=uG zT$xA>z~{-C5RA{McRMpug6$bn4`kU{xhX-?x-8)22ULY&^c$Ua@atmqO^2!=>=*04l8Pi&5>wUzn@PEDEftFnH+o#EgxFeA_ zQsORma)OnS`s4yjwJdOOHKGNUdI1;5G+$kX&0_BDyYyXTu5RzSXooe~Ndy@iSvd6Z0D*Q@%FgmgXf9p#*esoLYTMR9KzmI?pp0O-w5s>#qh`$InwPDjgWf_ekg42+i%sfw`$(mZ$7&3d=zMRyLaz0aj^{J zjd!@wUo)?7?x#$8`00n25cC~80JmgbVkORQU@$sy_TKH<)<`tU6N)t*nKWOU$~W%M zr_00R;{!>wIl1+Afqd~q`ZoVqc=F@NkMDNhTs=Hq-#uMrHvyU$Drv$gyfabEf6i24 z2vrah&;*1T;NCs_c>VMiT7}+w5W4h(1XLp$0ddt;i8@I(^_feU@&yk?-^s z`Iaa6PLC5V;{umc4tmT)co-DC6$7rF9AN3>P+!Z`&zKQI7~{Yp$_@}vHqQ#*z#Rv3 ziaQSK<=kUVJ+;K37I|8_pAu&pyc|EjfBC)UJ@EV#(>QML^+;ON@Gk$LFxH+*s zU*C||j-~eq$aBn;jt%5Fs#D}SLr}kjJjW1Qcr$8Bo})tKIi?mKV`7xP5BBDMi*WhI zVBUzX*A@C`Z%!a#gCh;zx8@8;Wu2LgB7qw(jo*&hwDRe!@-O ziR33wx$G~EGjuf&2PZt?9R@Ur@w*O&Oy4igaM95LuvNt!S<#-XnnkO&BF|y;u$$9Z zMF%}6;gU>u4P=0=!3MFGj$ggTdAPy7Snz91GRg#>txwjN0J#VRv{6V`OAjEh|G6-R?Ebs%`ARXKQ-^I{iN9BVMBN7lo zCWv?;1^8Q^ow*-go*f%I&x*Wx;tPa?km@z1gpl&TikNWg+`hz_lVK~@kQJGNCqux7 z;TixL25}8($pBcWd%`s&ZWret0*6wqa6ZTm!syB6E>aUQ>~j}BCttMWD&eK~H? zziRT2&kU^1{Ykk!p{Tbiw+E)nwWiUa@&K-P3r|WO%XzETjSheZafvm#%n2Zw*_(MT zu6bw=-{Ro-pg}5*}ljdkX)sj#Zy;iH_ zAY;tu1qoMV0e+T*E09N(d9`WA)yVjkCw;JD8OGRqF9#ynwhXE^BY7nU2j?TlPQN^T z#tf}e+7@k?3!U$5CbW{3tEJRw&E_Sk`CjjFO3?k~;bbB*Ogw(|L}Zc=T$zYW(w7H# z2`cEXlS3T%=gcb$4XH~x#LYgXc9#U2G& zB#TA(#_bxGbmIu78pJAE4lgCcwP`(MO5>Cot;c3J3fB7)XL@H;|;$+U4zihITisV+zQ2z6L zP?Z4(HTZsT2C$uSR22piQV>T-tz@|`gHSUbsv>+=jVI2J-+A3)qy~VUxc!>*38L?Y zx~`adoY!?_=LS*ubi7`)5hKJl7f`AHYFOtrV8aBRYseK#3_ja1e#0 zJkqN&9SrY)l&JKlkL|RSK=j$liWd3X7qu>#t5hfTktud2sQ+ZXZt?04XmxwIx?{Gw z176+aECXRQ?}wKyd}ykh;#d$d!&m%aNgWY@*g!Yl<+|ifd(^cetx zu8B#(O9`!>O4PSU)5ZdeV4Cjlcg3q$@`pF5ilJq95Dd%;r{OC zCl-pMaQYlU4y`ZEC_TOu-IZ^}YpX=pQ+fTToBR9kpC2;J2ev43n}>-iP{5_cy6raC zy60~xd7MsskJI27ZwBin6zp2jT9!H|R4aA^&V$aBCmv)1WnRs9VBnuu0hY*AlRHhYF5)u=RJ48U3*cJQvUfgu_H!nZ+n*F}15&GlSrHG0g5a4m?JPl<1w zmE^E)15(qAuduxs(@B7&?{WUU3Dextuz!r_MFul^i8X`6e!4G_EK-2W5JLqVX8G6l z39|Ol+XnxEDy2*k z54UY1JBIks`-pu#KmWtmhPW;=i|pe~77kmiA$iE&7>dwceS*!dMxcnIdAAnhkr+5y zY4-8rCACdh?Q{<S(dP?Lq24M>up&p(ACiFXmOk4ag3#sHHGug z0(aCM2wc^mGhNR}o+@~xG8;4V>{YBX@1Wm#yeIlo7j_x@YTat`L`-P|Gg)S;PP|Sy9*2AGFq|{$=mIy@76f$TIGLuW zi%2Di$5t?b8MQD<$7cN5l*r}gF76P6p7Q7QgM9C9;66-go!CoPVU zOLcsDcb}QA^;biZ04>F@()vjlvKOvyY8zUR5em+R`PE)S7D{&fU!gLl(6LY__+tZN zN_LKcWFhl(bw-3|gv3?~)w*+?0BXRp4SL7@oIQOm<1@?*8i|pU9-L!vJOSSg{FtEJ zilI8u5?GT^mrjNy6d{92vc{0Iw3ari@r#mOq=cag+Gqjv>~O8F10Nm6FX!O`9}Xx- zi+CwU|1<+vW9Hz{Os4TRLmfVKIzjD(QZt@jP9q`%>!YDdjig3h4u-;zO3CK%qZfdl;)eGk-zj#W{{X*>Fh+h(pP;Yo41IZU(hDa2>VcTWApAcW=3ZA zQ*y`}NXW%tD?WdXOhG-F3A9W=@dONM)!Dl4;LV8Q;k1|u=ShQz%HgA7psGg_>_YED zsd;I{DjVKnSiO7Uu+l8yStbxI{wg#GeLvGSnB!wDRaz%3Ur4q4(I8}N@Fk}?TgX=1 zWevxry+*C{)irNE%(et`U6K4eNi_O%H0qx{CHryK!9O_}m}S+U(!)hKun{pE7h2N8 z`EW3zs&iuSp2#LLKVJu>@!sd6k3a@>ko;G)obJzmRJ7eoKBB`RREP1FfTxUOhMPvm zeT%k~>{9ogeg`Xi)HW4_+-Q3YhuzD*C4ny|T`de_>QR~IIhMNWav1|H__rMj{gA4g z%H!FQB|eX(QEF6ta-f2WZ-$avtG{WA%o3R@YZc2454d4aRpqEX4QCl&NstbBWCvHN z1%(>X_56_d2A<4KD3j>(>HL=cC~H^;K9__T&f@--+ys)JC_RnlLK3nUIK_^so#sah zUJxATW!>jZH9h7>4vOL+MR60rwC`6%&x=FV2Su`JC8t^@dWyRB>dtdTZntKw+7*G{ zXf?guH+DJ|j&=B_kA4i|`g> z$>n5J`qQ@$rvAZ*itWM(qoFs@mZE=!RQdNlPnwEKNt&_3J3BE1Cpoa2WhHke51JXv z=|MSCx`GYaX5MdDEVZZtj{pxIy8NFrS&IO=8Yt3|1YZqQuObPd+vXBmVifJX-k74gYD*42y%ST6w4mfEMTpz1f<@7u`UmbYC#L*(0 z8m3dZ93ucD*CW65e&dk%CvDc+j)M~LE%2=nv>(oUZd2Ye5R|x3XaG(A&Dehyi9&5u z(BM7{_%&=flYsuKmwxu504E#48en>W27Sj|=G^MVXdV$x65Ymc z-+CXOCKqf=A7Kvn2s|w$;gyrYQELYCT~@E}fE-2`tmPCQ zkcJUO%K-^a40Y}f4e-t~1)KOA|27uKP)rKo6pT^~FqFMu0es4qu0UNemt5SH`K@g- zdUTR4P2F)uE!bgv&Z6TubTBFqHB4bBd&B|=Xt02r_3TUHYyLR0B$34bQSXB}_OHD2 zsz6VOoN4}1rC>U&LYJF*2QZU3_3`o_9ix<1s{?5s)T@%7)E6`~nouEpEt)p{*$%JX zLaiOi1dsDgb5zZ_$j9~X+d}<2wt8FTs#bz+)xT@WUs6*AbZVoX$-xA14n@Ya6^`#I zhGWuLj@a58N)fBV61cxZG_f#ya0Sr0N$6hW<3H}6nlrWo2Pwx13Dam+9@+z`7v?Jc zn-#aeh=fXbZ3Y8jhxfdRX47|@`Ua}@vP7gVLv_PeQI>{dvt7?LmDN{n%4KJ7a7 z)_ai;cM?^OmQUUCF%r3;HjMNa+Ta^sJVq{Lu9OLL@wZvBDDrWQJ+e(Gn`T&*GxV1F z^HScySG96LE1?hlAl8sEG6>Lmaf`YmqKbhLFo%;XLo5>%bt^5#_^jgtB+6Ma>(g#C z=@w|GR5{lY$(YPfb~=g2DV~L3sJ3F#cf%1UwGFm!Q$jrnap znWQjs423}*pLC8H3X_6_ng>Rr8a*+wYR|Kb`_(8qo;g2dyDk)0ibt27sW{)3=$3F! z>LhMLc5nilg0W2WGNtTcy{A@(mW(9}>NU>G4b>*mF@Ij3vo}#Fj{U#X;KFLjeh-i! zLvg=L%%;M@ygJZ<4+e<&%SKy<#!7Aq{#Uh12S)JEl|?J#e_PsSla0>X5N}Rn0s>Q{ zWrfA$xUNyKMj?)@5%3zLZM*B51zld9)2xv((w;$@H$0swq-%6tZT5r?_^_QAM9EHs zKXf*o*~s)gxh>l4bA+x2KU_bMpN}1PFlBCIFlh9i>Xm6YAsw}2K{yntqd-+Fs=l_l ztyU&?3uM$)@AtJ|P=~aHq2Sg)Nbpj*!lPdSm$o<}fALlW1g-Y#{`+l>u#G{R*TYSDHAj`c6CnPb$Snl84 zO=IP4%wS6>m2mM8meY;{yux1rrZn_eKNTYmLuOq$YOA-g8heW&GD)@CowJifc>M}oHsA<5MFzI7^p9lc}jz|mq3F}s-zBz`rO5Bgab zVBtK)l6-?6RU5q!O~k<&aIM&;m~iKyp*b>p2x~o}vla~>xP-2kU%kAjebp$Gr#2=+qnJ1dIFz(hox0^mP1}6i zdpy%$C-fgtw{%-ymJV+7YIkhv@mpw9B6MQ{jTSdj){QMYIis>Nt5M$A>}z*Z6k_Vs z`IO~E0ZB)@UdCRnv6^QwIUaH8zF72hRSqF-f$)E=q!lQnD$FR@@bg$0VIobk@IW75 z(3tZJ*&qkjW*eV+D}rdkiV6W2C>Ge;*t99E`IJ%REhj1I8H0T2tD51a(YQol^U$_e zNqG5l8AY)@(p{0Ms~c42Yj-PwayYD<&e-vdd%`T}IpVdJc0u#mq0 z6TB?@Nng1CwwP|6W!BGuA8f>7L?lPV`--Au^D_f^0Ir|oEZE5L6ATawHV_j57jX+F zph{N5+}zQ@Vc|xZ9($j7)J%Y3P#nC<=UdKgtca?~RkS6M0g)U+);51UuIJJXOsv3T%m4+jCog*^CZlqd~SOW%T*Lr6r=DZQVo*uN7HfJ$i;Bm>fD=n zG7CDGJPgm5MD2l-;=Lu99Cl;kQ!|WoGFLRhgj0y~(JQB=wOsz}GXlMH7(u}&RX=xe zMwo~^X^IS{)VjbkqsOw2hK4CxRttn~I#$QHIyt}AHw%_I@`44GMaP46fZ`zGxT=hF z+#nJuKzd95iv)h)YPmx`)FTg|#lhAxN`D8S{u4nSiiS3OWPu8OF^opsEb;R05b1P= zb2YD4A<)_EeT52h1gT=(R3xf75^F=RpcYWq1aoQ%g}ZMo8!~JN;^${|+=^tkZBddM zX;jC}VI&QYHCc6~O@j}~Lj{ScWQQZwZ<=Co>?n9f))32YQ8Ofb=fn@|>EZ=6WVd|B???yz7VqD*I%38uoC-Ju{u%pQx}ye z!LX(To73swsQ@xGaQhdVuysD)gOCo?j6%K+mE!{S-|y=-{D1Cx%=m@A5G^ z8(2&%`K^eJ4mZORRq1Op)4QdofZuxTgpWj`nP6$rT4^$wG$f7Xv(>~TQ|tQLWNlH3 zvobu7{b-s(0%PRt23JR9w}*AZL>BwrYD$wbMSyLD)eTKKx_TurUMB+pjHNnMWW#F0 ztLDTV(+$-#3aUcS;1qM$l7qRIQK7_UD5CE#e#}oR!}T6L|910bmAz}jG*y>rVyxKy zo0*B7mIP?7)58=Qlva**aEhiq?g}bcd<3Gz_AsLG7h6wRC3MJjFmx**LXACL3bvr==~Ed*C*@cOGrbRrJX>fqeC zE`Z$ZVh)Xcq)_N|=xSoGumnkv{i;0eesbcf;C}LGHQ5HX^|n`xPs3Z!0c+_u1fR3;J0RMsoJforU*r-dE@&<`P>V<>DMEOCYO2&E#dd&Jjb!?F!+dC8 zc7@D-xN%a?mXM)G9ncxn{G+YV;ihkWV(&%9VX;gQk>*$%#FLPi^Z<9vP~6YtFZOId z!DaZ7{rD@(`h@Ye)wsC8`Z|-K)@etw@a=;wW3lubzPwQ}(Sg(iv8z#ovh(XHHwRf% zA!dD%^Kj`&Q;-!@9LgEgIwOwLU7|-ytCzDS`CGc)+9AU}oX9%GKbWUgcS2R`sqo%{ zru!k6=?++Xts_9`Z6_edv&eu^7Po*l)S)*qBGTK_NlMteg<*&_@>)PFh?9MU^MPB#?a0^v%Rn z8bdx!AG#u4cu*TvRk3)Ip^qG*<@EY~xuG;p$c3RF;6Lq+GGrTvmKf7n!VoIz>r zGo~V0l)>+~!ZpR0DznBv`0WixVZmg>Se{01fka*>4WWqu$dZ13I@!3$IXq13@%JT^ zm&dk#G29#dYeWVh(F;TEMNPDWy$-yL#bXF%eS{jy=Vl-d+|>_OM%At^>8%L{$H=(M$eh>(PW_^q$22BTZz7%ZQpfQDUhu{U-V|8g zf6}#%+W(cpe`@+w^B|?GF`5;u?GbC6(5ZW!%GokUzD&KEdR@bU;e5wR<^5|r1x*iFx1*M;U-$$^{IK$xsgu{WMfGG-9QNBpNA(HUhXtF{$*o>561NN zxe7Z78>S1?(WVPG9)MwH#^Sww-RGX(^@ji5ru%kl;r482;dZlheQ$kV0G9XU=I_IE zEM^$04F;u7s{^L@``BDl&pqpg$M#C*R*>s8qSQhY)hT+A`)jQ;3kM2-II*j)Lq@C>LdlOSxeQZ-f9kead zbBws>(*4m_!cs@_nsajda&EUaO5Wpu2xU{*S}|VO;i%r`Q7Z&I5@kkICUTwTAH3aW z;qP+WRZ_$>(0y_yGoU~@Jw$QWf1Udaby%TNck{~<@e~8v+Ml$u1ViOe6YyGl&Eq)- z5hJQMqAK=h(yG`v_TxgO&|O6PoMI-?)fdgK4;{OFH<=?h(TC)yKKd?H+v7 znvi#d{Jj@4DhN6Q_NRx;`P)kl=g z@4Qe|3)@=7YT^cC6}l40LV~88aZ*fHF>0`v^3_rB^fst^_26>obCVsvz3zvBx2x-5 z(Hcwf-dP=ll{bQR4MIf1F%hut_WWGEDc_($7Ow-h(>q^MgT_z)Tk@Bdh_*~A_}|#| zmPXY=f*@rw+uzFa8vSO)(`I&SWdub+(!MCAj{RFtn1fYvpvI(eomw@9o5u3Bs@7+Ts>$J#Jx~xZoIArMeM>C;(NyVX zuYG}WMyv*1B;aG~)5n7c_FN(D_${#3@3}Getc&Kzkt3*}&0Ktq^pJ*W$@Z$&@a<~{ zwaqj1oh6Z&1c*q!radZT0PJdHD{SN~zF@<0v^(;;w6jojNg2{X9uz2SQhmR$cKCXO z`&$%|grvQ_-9yw@dyYMf%{L4FFBd!;e{wSmWmqrC_r$)^9UxNM5=zs{a~qcAE)r`! ze?i&<8ojr!@5?sJZHyr$G~j2c4mEH`B?gHqH$!oJ*`F2HFpT4yHV@A}*jx^5W-&Y& z-bzEia?uLW5%uK{&r$A|JOAvnT81%#Ix9~JJp2lH>a;yxjYB6~KtPK##WFGy79D`g#(Ms% zv=@GhMoK0JcZdYkxsnxoxVg0GJiVue>X=Lmxrjqe#Q4^UIdrP?^-NVWbFE%MHlNzO z*Ti?k5@n2ZK16{0^vTHEx*$|36Z+mP{SHC?cVQ;!l^8+XvI+CrxE0O4bNRAaU(^WC zv@m6+zKGV1`-;Jnv73)uVcm#eH7C3ChNmh&CLsDKDsAoAr8hIpQ8Ctv-9s|YETABD zq43g(xV|Bf5A5#7hZuLnkRd~UvNM7u4};GuQ4|S?$1|AethwO0Ac(;CkcN!^8crYs z6q=)C^pO269XZY-wTTzzG$!$Zn&N_Yi1VzT#~FzeZWil$88}ZyHGoLPM;7PC@?h@_ zVgf<>@s)W!Uh*OO;sMFX#By|9lssuxRwX8B4zo{za!ROlN$CL-Y2!6;I{J>PlOl}= zXou7d6cgSfTx4~FT7~K+D^p+DLF1URX60yg9<}MUmhk$MCB8V|i1{k^l3fh`h%C9* zbgTMGV}XN3a0T(8LI610Xn86PUvEtF)EWO-H`v$b?iA!S?E^)Z{ar)2VG^0MgiGDi zCBgg3^c_Mto#X%hd(k?JMkI|~b-dV+iw33b1HVO|r4WEaXQ4L8Orz*+JqR9x7KL7* zlW&0%In<>@^HTO97<$%=PbQNR#7pn`Rg@-I=V)u%v@fsBkDLaKnM9Fml!tXgw2f4% zmw4Kx>tSIwq5bufjyV{3RyKvQlPId9fwUY>vr%WlRXc{!cQzCDrImuT-1c*XNTDr% zHXOOpJnVkDB3t#$1lHcss4~L?=R{s*OCl0gKh7~qgfkj7^TB*&I+c=9&Ckn`4(%JqLbWXnWCZQ-BN{f3REjWo>b^^d2zqG zTquoV-!i7=z;h}ffASV2nE*W|HfjMl`=EOo+o}}8%!qb2YBLb_O|n>?4lC;o555WU z(2)X8!d8>TiLB#R&?M$FXm7aFCug8ec3H?wqVkNa2J6)rwMw*e7OHkObex-wX{SQf z*dPNcUmCi1r<&t6Cnv*KpO|@hb$=HJ?A=cAd-h*Ex%iq7A^Q>%RM7hFdG!i_g_&us z;+-sbMZyS1$ZXXVZ3^$Y-9E6%4T4zf|u!KEk%ek|!_fd7&19=NhfO9GFhzZJr zq_$@q(DMTbkG8%0cK!gF`%EKB<|iadYA4qA?$^31Mmim!LH5z4l`KOh?5KR5;%sb> zcqA?E3JK+U7{LRVcC6Pt{Cc<=sj`2{)Kp6W*N<2`{1qMgljhKWyXb^D^p8X=7CJPW z@*d~2t=%PtPxqI!#R%`ql|Q}Gu;HG9=Z+anKb=4}MIiw5@XSD1uHHBWM;V-XMX z-Vx3&F>=hflSQ)5+we8b{A4~guTf9N?#NF*4)*3X9<-dq+(tSP&KvSYurU%CRHb7D z()bXOND?uzEN+KpQ~RB&UZWUln~gV#R=}fDGd>R*#QRw|D~*tc<$F-1oa)%bAe4QO zA_F&$!26%Y_H;@M5F-oAMpUukj0@(3P8S^$ls zG<)=PnnuZf>RBi@H)JH=i327DgjtAywh~WcO$XRf(1V#>hh}KUD-pU~bJ^xS}`F zLLoGES#l>as)&icLm><8!3IOquDEFeSlOE2weXT=*1&_uT$79CT^DE2jf3B;;#&}d z3{nzn5+9W=(ySW5olps&Bg+&TPGpz@)mclPqssFI`53IeJ`hr*YNuRXh$>tw8x%aN z$eCc*0d;8{2^bRXDsVDZAjpF8qxcjy@M=9i2(ixvrn8*iwaYgRgqg`$e6wYv9_}l- zrVtP!fTUj@6oY2tR5c2S$fsxYB`NS0F(P%^RAyJX#-Ns#>3Ay(O2#B69r}?$#ShT+ zk_njGAHaK-dtXcq@7AuKj>oahgM9d68xV(sr;D`Kss zm>uL~HF+3sxHp$a=%S(=-!Vlo>pHr-T&SIbT(iL!GM=xGkH-#>Rkiol=X=&igN~YN zvip`WOw99y(t$nA_lJ~@i-9R4zuTqr%604aL$~KFOg|w#f!LRwu#<^W_Ve87@0}g* zmylLJ_tx&`^MNzbms5d_zV9CHogZWk2k*5H43zK7KL5@G?&oWx_bZ zH!(30;mtfhACLF_KRzCBw@U+IuX|^C2=8gNnl1e2aUVn9?{POZ_OW3%H5VK8d48|0 zZZ!`3yMM0SJ-ps-uO{?E-!qvRZ z#Y6#!W3aYS9OTZs0ka-~joF)e@OtYc5WS`u1>*@2~JH~&Q=eA&vS-<;Aa<=YMaxr*t6u+fgoZdO{i%1^% zy&Iu{sN%kBYe&0ZCsOJzMDCDG^#Pl_;2Ep*-N?`5xaUw#EQg*f{RwESa)9Xp~+^mO>5_2E{2gV%BL znH=uN^X~OmM!Qj^qqt4kCa+JvuN2Ou8$HF&2oTw;bLJj;Uw-zQIOXN(en<^=0NaaK zNVH#U}v9_jhggX_k^v_(hYf`Vt8$q{XXUOgo7JKHTptw#@VRBY77 zGq14c4V(uhOjHD`VLJm^(vGY~O?hd)qqkLs7FANFEyX+Y?$-knt3&lPBY! z0gl?}Kut-u%7P=SGE-i>@90fcp@o$NsY@!(yo>dK$m&oX4avDBl?f|`c7T;Oy0oSw z>;FhDsVqoansw%d)B`fBL-jQz7nWuxtSs6Ap87HGuqq1BFl4a*pHPSBL$m`0^N?&