/* OpenWebSpider * * Authors: Stefano Alimonti AND Stefano Fantin * Version: 0.7 * E-Mails: shen139 [at] openwebspider (dot) org AND stefanofantinguz@yahoo.it * * * This file is part of OpenWebSpider * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * * RankSVM-style learning exports the linear feature weights used by the ranker. * Monte Carlo validation defines the uncertain domain as ranker weights, page * features, and coupled per-round scenario shocks; it uses low-discrepancy * scenario samples plus uncertainty-weighted local perturbations, runs the * deterministic scorer on each accepted realization, then aggregates mean * score, variance, standard error, and rank stability. * Ranking remains content-sensitive: changes in extracted text, metadata, or * link context can legitimately change the final order. */ #ifndef __RANK #define __RANK /* Requires MySQL/MariaDB headers included before this file (MYSQL_RES, MYSQL_ROW, etc). */ /* Optional compile-time hint for contributors: enable with -DOWS_RANK_WARN_MISSING_MYSQL_HEADERS=1 (or define OWS_HAVE_MYSQL_HEADERS before including rank.h). */ #ifndef OWS_RANK_WARN_MISSING_MYSQL_HEADERS #define OWS_RANK_WARN_MISSING_MYSQL_HEADERS 0 #endif #if OWS_RANK_WARN_MISSING_MYSQL_HEADERS && !defined(OWS_HAVE_MYSQL_HEADERS) #if defined(__GNUC__) || defined(__clang__) #warning "rank.h expects MySQL/MariaDB headers included before it (define OWS_HAVE_MYSQL_HEADERS where mysql.h is included)" #endif #endif /* Self-contained header dependencies. */ #include #include #include #include #include #include /* GCC/Clang __atomic builtins required for the atomic path. */ #if defined(__GNUC__) || defined(__clang__) #define OWS_HAVE_ATOMICS 1 #else #define OWS_HAVE_ATOMICS 0 #endif struct ows_rank_weights_cfg { double w_level; double w_title; double w_text; double std_level; double std_title; double std_text; int loaded; }; #define OWS_MAX_RANK_PROFILES 16 #define OWS_RANK_PROFILE_NAME_MAX 32 #define OWS_INTENT_CACHE_TTL_DAYS 7 #ifndef OWS_URL_INTENT_CACHE_TTL_DAYS #define OWS_URL_INTENT_CACHE_TTL_DAYS 14 #endif #ifndef OWS_URL_CACHE_BATCH_MAX #define OWS_URL_CACHE_BATCH_MAX 100 #endif #define OWS_URL_CACHE_FLUSH_FINAL 0 #define OWS_URL_CACHE_FLUSH_BY_COUNT 1 #define OWS_URL_CACHE_FLUSH_BY_75 2 #define OWS_URL_CACHE_FLUSH_BY_CAP 3 #ifndef OWS_PROFILE_INIT_SPIN_MAX #define OWS_PROFILE_INIT_SPIN_MAX 50000000U #endif #ifndef OWS_RANK_MC_ROUNDS #define OWS_RANK_MC_ROUNDS 48 #endif #ifndef OWS_RANK_MC_ROUNDS_MIN #define OWS_RANK_MC_ROUNDS_MIN 8 #endif #ifndef OWS_RANK_MC_ROUNDS_MAX #define OWS_RANK_MC_ROUNDS_MAX 256 #endif #ifndef OWS_RANK_MC_STD_PENALTY #define OWS_RANK_MC_STD_PENALTY 0.35 #endif #ifndef OWS_RANK_MC_STD_PENALTY_MAX #define OWS_RANK_MC_STD_PENALTY_MAX 0.15 #endif #ifndef OWS_RANK_PROFILE_STD_WEIGHT_RATIO #define OWS_RANK_PROFILE_STD_WEIGHT_RATIO 0.80 #endif #ifndef OWS_RANK_PROFILE_STD_COMPONENT_FLOOR #define OWS_RANK_PROFILE_STD_COMPONENT_FLOOR 0.02 #endif #ifndef OWS_RANK_PROFILE_STD_TOTAL_MAX #define OWS_RANK_PROFILE_STD_TOTAL_MAX 0.35 #endif /* Enable this if you want a one-shot warning when init wait times out. */ #ifndef OWS_RANK_DIAG_FALLBACK #define OWS_RANK_DIAG_FALLBACK 0 #endif /* Optional lightweight duplicate-PID diagnostic (modulo hash), for debugging only. */ #ifndef OWS_RANK_DIAG_DUP_PID #define OWS_RANK_DIAG_DUP_PID 0 #endif #ifndef OWS_RANK_MAX_PAGES_PER_HOST #define OWS_RANK_MAX_PAGES_PER_HOST 1000000L #endif #define OWS_RANK_GUARD_RET(cond, retv) do { if(!(cond)) return (retv); } while(0) #define OWS_RANK_GUARD_VOID(cond) do { if(!(cond)) return; } while(0) #define OWS_RANK_GUARD_GOTO(cond, label) do { if(!(cond)) goto label; } while(0) #define OWS_RANK_GUARD_SQL_RET(wr, cap, retv) do { if(!ows_rank_snprintf_ok((wr), (cap))) return (retv); } while(0) #define OWS_RANK_GUARD_SQL_VOID(wr, cap) do { if(!ows_rank_snprintf_ok((wr), (cap))) return; } while(0) #define OWS_RANK_ALLOC_ARRAY_OR_GOTO(ptr, count, type, label) do { (ptr) = (type *)ows_rank_alloc_array((count), sizeof(type)); if((ptr) == NULL) goto label; } while(0) #define OWS_RANK_FREE_PTR(ptr) do { if(ptr) { FREE(ptr); (ptr) = NULL; } } while(0) #define OWS_RANK_FREE_PAGE_ARRAYS() do { \ OWS_RANK_FREE_PTR(ids); \ OWS_RANK_FREE_PTR(ranks); \ OWS_RANK_FREE_PTR(levels); \ OWS_RANK_FREE_PTR(titleLens); \ OWS_RANK_FREE_PTR(textLens); \ OWS_RANK_FREE_PTR(pageWLevel); \ OWS_RANK_FREE_PTR(pageWTitle); \ OWS_RANK_FREE_PTR(pageWText); \ OWS_RANK_FREE_PTR(pageStdLevel); \ OWS_RANK_FREE_PTR(pageStdTitle); \ OWS_RANK_FREE_PTR(pageStdText); \ OWS_RANK_FREE_PTR(pageOrder); \ OWS_RANK_FREE_PTR(pageChaos); \ OWS_RANK_FREE_PTR(accScores); \ OWS_RANK_FREE_PTR(accSqScores); \ OWS_RANK_FREE_PTR(minScores); \ OWS_RANK_FREE_PTR(maxScores); \ OWS_RANK_FREE_PTR(mcCounts); \ } while(0) struct ows_rank_profile_cfg { char name[OWS_RANK_PROFILE_NAME_MAX]; struct ows_rank_weights_cfg w; }; struct ows_rank_profiles_cfg { struct ows_rank_profile_cfg profiles[OWS_MAX_RANK_PROFILES]; int count; int loaded_from_file; }; struct ows_rank_profile_pick { struct ows_rank_weights_cfg w; char name[OWS_RANK_PROFILE_NAME_MAX]; char source[64]; double confidence; int conf_file_loaded; }; static void ows_rank_weights_defaults(struct ows_rank_weights_cfg *cfg) { OWS_RANK_GUARD_VOID(cfg != NULL); cfg->w_level = 0.52; cfg->w_title = 0.20; cfg->w_text = 0.28; cfg->std_level = 0.08; cfg->std_title = 0.05; cfg->std_text = 0.07; cfg->loaded = 0; } static void ows_trim_inplace(char *s) { size_t i, n; OWS_RANK_GUARD_VOID(s != NULL); n = strlen(s); while(n > 0 && (s[n-1]=='\r' || s[n-1]=='\n' || s[n-1]==' ' || s[n-1]=='\t')) { s[n-1] = '\0'; n--; } i = 0; while(s[i]==' ' || s[i]=='\t') i++; if(i > 0) memmove(s, s+i, strlen(s+i)+1); } static double ows_clip(double v, double lo, double hi) { if(lo != lo || lo > 1.0e100 || lo < -1.0e100) lo = 0.0; if(hi != hi || hi > 1.0e100 || hi < -1.0e100) hi = lo; if(hi < lo) { double t = lo; lo = hi; hi = t; } if(v != v) return lo; if(v < lo) return lo; if(v > hi) return hi; return v; } static double ows_absd(double v) { return (v < 0.0) ? -v : v; } static int ows_rank_bad_number(double v) { return (v != v || v > 1.0e100 || v < -1.0e100); } static int ows_rank_parse_double(const char *s, double fallback, double *out); static int ows_rank_parse_int(const char *s, int fallback, int *out); static int ows_rank_snprintf_ok(int wr, size_t cap) { return (wr >= 0 && cap > 0 && (size_t)wr < cap); } static int ows_rank_guard_sqlbuf(char *buf, size_t cap) { return (buf != NULL && cap > 1); } static int ows_rank_guard_host_id(int host_id) { return host_id > 0; } static int ows_rank_guard_output(char *out, size_t cap) { return (out != NULL && cap > 0); } static int ows_rank_guard_page_count(long n) { long maxPages = OWS_RANK_MAX_PAGES_PER_HOST; if(maxPages <= 0) maxPages = 1000000L; return (n > 0 && n <= maxPages); } static int ows_rank_guard_mysql_fields(MYSQL_RES *res, unsigned int min_fields) { return (res != NULL && mysql_num_fields(res) >= min_fields); } static MYSQL_RES *ows_rank_guard_tmp_result(MYSQL_RES **tmpRes) { if(tmpRes == NULL) return NULL; return *tmpRes; } static int ows_rank_guard_nonempty_string(const char *s) { return (s != NULL && s[0] != '\0'); } static int ows_rank_guard_profile_name(const char *profile) { return ows_rank_guard_nonempty_string(profile); } static int ows_rank_guard_batch_buffer(const char *buf, size_t cap) { return (buf != NULL && cap > 1 && strlen(buf) < cap); } static int ows_rank_guard_row(MYSQL_ROW row) { return row != NULL; } static int ows_rank_guard_row_text(MYSQL_ROW row, unsigned int idx) { return (row != NULL && row[idx] != NULL && row[idx][0] != '\0'); } static const char *ows_rank_row_text_or_null(MYSQL_ROW row, unsigned int idx) { return ows_rank_guard_row_text(row, idx) ? row[idx] : NULL; } static int ows_rank_row_parse_int(MYSQL_ROW row, unsigned int idx, int fallback, int *out) { OWS_RANK_GUARD_RET(out != NULL, 0); if(!ows_rank_guard_row(row)) { *out = fallback; return 0; } return ows_rank_parse_int(row[idx], fallback, out); } static int ows_rank_row_parse_double(MYSQL_ROW row, unsigned int idx, double fallback, double *out) { OWS_RANK_GUARD_RET(out != NULL, 0); if(!ows_rank_guard_row(row)) { *out = fallback; return 0; } return ows_rank_parse_double(row[idx], fallback, out); } static int ows_rank_guard_positive_int(int v) { return v > 0; } static int ows_rank_guard_nonnegative_int(int v) { return v >= 0; } static int ows_rank_guard_batch_count(int *count) { return (count != NULL && *count >= 0); } static int ows_rank_guard_rounds(int rounds) { return rounds >= OWS_RANK_MC_ROUNDS_MIN && rounds <= OWS_RANK_MC_ROUNDS_MAX; } static int ows_rank_guard_level(int level) { return level > 0 && level <= 10000000; } static int ows_rank_guard_length_int(int len) { return len >= 0 && len <= 10000000; } static int ows_rank_guard_unit_interval(double v) { return !ows_rank_bad_number(v) && v >= 0.0 && v <= 1.0; } static int ows_rank_guard_sql_piece(int wr, size_t cap) { return ows_rank_snprintf_ok(wr, cap); } static int ows_rank_guard_positive_long(long v) { return v > 0; } static int ows_rank_guard_positive_double(double v) { return !ows_rank_bad_number(v) && v > 0.0; } static int ows_rank_copy_string(char *dst, size_t cap, const char *src) { if(!ows_rank_guard_output(dst, cap)) return 0; if(src == NULL) src = ""; return ows_rank_snprintf_ok(snprintf(dst, cap, "%s", src), cap); } static const char *ows_rank_string_or_default(const char *s, const char *fallback) { if(ows_rank_guard_nonempty_string(s)) return s; return fallback ? fallback : ""; } static void ows_rank_reset_buffer(char *buf, size_t cap) { if(ows_rank_guard_output(buf, cap)) buf[0] = '\0'; } static int ows_rank_guard_mysql_escape_input(const char *src, size_t dst_cap) { size_t n; if(src == NULL || dst_cap == 0) return 0; n = strlen(src); return n <= ((dst_cap - 1U) / 2U); } static int ows_rank_escape_string(MYSQL *mysql, char *dst, size_t dst_cap, const char *src) { if(mysql == NULL || dst == NULL || !ows_rank_guard_mysql_escape_input(src, dst_cap)) return 0; mysql_real_escape_string(mysql, dst, src, (unsigned long)strlen(src)); return 1; } static int ows_rank_guard_buf_append_args(char *buf, size_t *len, size_t cap, const char *txt) { return (buf != NULL && len != NULL && txt != NULL && cap > 1 && *len < cap); } static int ows_rank_guard_rank_batch_args(char *sqlQuery, char *caseBuf, size_t *lenCase, char *idBuf, size_t *lenId, int *batchCount, const char *rowCase, const char *rowId) { return (ows_rank_guard_sqlbuf(sqlQuery, (size_t)MAXQUERYSIZE) && caseBuf != NULL && lenCase != NULL && idBuf != NULL && lenId != NULL && ows_rank_guard_batch_count(batchCount) && rowCase != NULL && rowId != NULL); } static int ows_rank_parse_double(const char *s, double fallback, double *out) { char *endp; double v; OWS_RANK_GUARD_RET(out != NULL, 0); if(s==NULL) { *out = fallback; return 0; } while(*s==' ' || *s=='\t' || *s=='\r' || *s=='\n') s++; if(*s=='\0') { *out = fallback; return 0; } errno = 0; v = strtod(s, &endp); if(endp==s || errno==ERANGE || ows_rank_bad_number(v)) { *out = fallback; return 0; } while(*endp==' ' || *endp=='\t' || *endp=='\r' || *endp=='\n') endp++; if(*endp!='\0' && *endp!='#' && *endp!=';') { *out = fallback; return 0; } *out = v; return 1; } static int ows_rank_parse_int(const char *s, int fallback, int *out) { char *endp; long v; OWS_RANK_GUARD_RET(out != NULL, 0); if(s==NULL) { *out = fallback; return 0; } while(*s==' ' || *s=='\t' || *s=='\r' || *s=='\n') s++; if(*s=='\0') { *out = fallback; return 0; } errno = 0; v = strtol(s, &endp, 10); if(endp==s || errno==ERANGE || v < (long)INT_MIN || v > (long)INT_MAX) { *out = fallback; return 0; } while(*endp==' ' || *endp=='\t' || *endp=='\r' || *endp=='\n') endp++; if(*endp!='\0' && *endp!='#' && *endp!=';') { *out = fallback; return 0; } *out = (int)v; return 1; } static int ows_rank_mysql_rows_to_long(unsigned long long rows, long *out) { OWS_RANK_GUARD_RET(out != NULL, 0); if(rows > (unsigned long long)LONG_MAX) { *out = 0; return 0; } *out = (long)rows; return 1; } static int ows_rank_safe_add_int(int a, int b, int *out) { OWS_RANK_GUARD_RET(out != NULL, 0); if((b > 0 && a > INT_MAX - b) || (b < 0 && a < INT_MIN - b)) { *out = (b > 0) ? INT_MAX : INT_MIN; return 0; } *out = a + b; return 1; } static int ows_rank_safe_page_rank(int hostrank, double rankScore, int depthBonus) { int scorePart; int rnk; if(hostrank < 0) hostrank = 0; rankScore = ows_clip(rankScore, 0.0, 1.0); scorePart = (int)(rankScore * (double)MAXPRLEV); if(scorePart < 0) scorePart = 0; if(depthBonus < 0) depthBonus = 0; (void)ows_rank_safe_add_int(hostrank, 1, &rnk); (void)ows_rank_safe_add_int(rnk, scorePart, &rnk); (void)ows_rank_safe_add_int(rnk, depthBonus, &rnk); if(rnk < 1) rnk = 1; return rnk; } static int ows_rank_alloc_count_ok(long n, size_t elem_size) { size_t max_size = (size_t)-1; if(n <= 0 || elem_size == 0) return 0; return (unsigned long)n <= (unsigned long)(max_size / elem_size); } static void *ows_rank_alloc_array(long n, size_t elem_size) { if(!ows_rank_alloc_count_ok(n, elem_size)) return NULL; return malloc((size_t)n * elem_size); } static double ows_rank_profile_std_ratio(void) { double v = OWS_RANK_PROFILE_STD_WEIGHT_RATIO; if(ows_rank_bad_number(v) || v <= 0.0) return 0.80; return ows_clip(v, 0.05, 2.0); } static double ows_rank_profile_std_floor(void) { double v = OWS_RANK_PROFILE_STD_COMPONENT_FLOOR; if(ows_rank_bad_number(v) || v < 0.0) return 0.02; return ows_clip(v, 0.0, 0.25); } static double ows_rank_profile_std_total_max(void) { double v = OWS_RANK_PROFILE_STD_TOTAL_MAX; if(ows_rank_bad_number(v) || v <= 0.0) return 0.35; return ows_clip(v, 0.01, 1.0); } static void ows_rank_profile_guard_weights(double *wLevel, double *wTitle, double *wText) { double sum; if(wLevel==NULL || wTitle==NULL || wText==NULL) return; if(ows_rank_bad_number(*wLevel)) *wLevel = 0.0; if(ows_rank_bad_number(*wTitle)) *wTitle = 0.0; if(ows_rank_bad_number(*wText)) *wText = 0.0; *wLevel = ows_clip(*wLevel, 0.0, 1.0); *wTitle = ows_clip(*wTitle, 0.0, 1.0); *wText = ows_clip(*wText, 0.0, 1.0); sum = *wLevel + *wTitle + *wText; if(sum <= 0.0 || ows_rank_bad_number(sum)) { *wLevel = 0.52; *wTitle = 0.20; *wText = 0.28; return; } *wLevel /= sum; *wTitle /= sum; *wText /= sum; } static double ows_rank_profile_std_cap(double weight) { double floor = ows_rank_profile_std_floor(); double cap = ows_clip(weight, 0.0, 1.0) * ows_rank_profile_std_ratio(); if(cap < floor) cap = floor; return ows_clip(cap, floor, 0.5); } static void ows_rank_guard_std_total(double *stdLevel, double *stdTitle, double *stdText, double totalMax) { double stdSum; double stdScale; if(stdLevel==NULL || stdTitle==NULL || stdText==NULL) return; if(ows_rank_bad_number(*stdLevel)) *stdLevel = 0.0; if(ows_rank_bad_number(*stdTitle)) *stdTitle = 0.0; if(ows_rank_bad_number(*stdText)) *stdText = 0.0; *stdLevel = ows_clip(*stdLevel, 0.0, 0.5); *stdTitle = ows_clip(*stdTitle, 0.0, 0.5); *stdText = ows_clip(*stdText, 0.0, 0.5); if(ows_rank_bad_number(totalMax) || totalMax <= 0.0) totalMax = 0.35; totalMax = ows_clip(totalMax, 0.01, 1.0); stdSum = *stdLevel + *stdTitle + *stdText; if(ows_rank_bad_number(stdSum) || stdSum < 0.0) { *stdLevel = 0.0; *stdTitle = 0.0; *stdText = 0.0; return; } if(stdSum > totalMax && stdSum > 0.0) { stdScale = totalMax / stdSum; if(ows_rank_bad_number(stdScale) || stdScale < 0.0) { *stdLevel = 0.0; *stdTitle = 0.0; *stdText = 0.0; return; } *stdLevel *= stdScale; *stdTitle *= stdScale; *stdText *= stdScale; } } static int ows_rank_profile_amplitude_ok(double stdLevel, double stdTitle, double stdText, double totalMax) { double stdSum; if(ows_rank_bad_number(stdLevel) || ows_rank_bad_number(stdTitle) || ows_rank_bad_number(stdText)) return 0; if(ows_rank_bad_number(totalMax) || totalMax <= 0.0) return 0; if(stdLevel < 0.0 || stdTitle < 0.0 || stdText < 0.0) return 0; if(stdLevel > 0.5 || stdTitle > 0.5 || stdText > 0.5) return 0; stdSum = stdLevel + stdTitle + stdText; if(ows_rank_bad_number(stdSum) || stdSum < 0.0) return 0; return stdSum <= (ows_clip(totalMax, 0.01, 1.0) + 0.000000001); } static void ows_rank_limit_profile_amplitude(double wLevel, double wTitle, double wText, double *stdLevel, double *stdTitle, double *stdText) { double gLevel; double gTitle; double gText; double totalMax; if(stdLevel==NULL || stdTitle==NULL || stdText==NULL) return; gLevel = wLevel; gTitle = wTitle; gText = wText; ows_rank_profile_guard_weights(&gLevel, &gTitle, &gText); *stdLevel = ows_clip(*stdLevel, 0.0, ows_rank_profile_std_cap(gLevel)); *stdTitle = ows_clip(*stdTitle, 0.0, ows_rank_profile_std_cap(gTitle)); *stdText = ows_clip(*stdText, 0.0, ows_rank_profile_std_cap(gText)); totalMax = ows_rank_profile_std_total_max(); ows_rank_guard_std_total(stdLevel, stdTitle, stdText, totalMax); *stdLevel = ows_clip(*stdLevel, 0.0, ows_rank_profile_std_cap(gLevel)); *stdTitle = ows_clip(*stdTitle, 0.0, ows_rank_profile_std_cap(gTitle)); *stdText = ows_clip(*stdText, 0.0, ows_rank_profile_std_cap(gText)); ows_rank_guard_std_total(stdLevel, stdTitle, stdText, totalMax); if(!ows_rank_profile_amplitude_ok(*stdLevel, *stdTitle, *stdText, totalMax)) { *stdLevel = 0.0; *stdTitle = 0.0; *stdText = 0.0; } } static int ows_try_mark_once(int *flag) { int expected; if(flag==NULL) return 0; #if OWS_HAVE_ATOMICS expected = 0; return __atomic_compare_exchange_n(flag, &expected, 1, 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE); #else /* Non-atomic fallback: not thread-safe, used only when atomics are unavailable. */ if(*flag) return 0; *flag = 1; return 1; #endif } static void ows_rank_weights_clip(struct ows_rank_weights_cfg *cfg); static void ows_cpu_relax_backoff(unsigned int *spin) { unsigned int n = spin ? *spin : 0U; unsigned int k; unsigned int reps; #if defined(__x86_64__) || defined(__i386__) reps = (n < 128U) ? 1U : ((n < 4096U) ? 8U : 64U); for(k = 0; k < reps; k++) __builtin_ia32_pause(); #elif defined(__aarch64__) || defined(__arm__) reps = (n < 128U) ? 1U : ((n < 4096U) ? 8U : 64U); for(k = 0; k < reps; k++) __asm__ __volatile__("yield"); #else (void)n; #endif if(spin) (*spin)++; } static void ows_rank_diag_fallback_once(unsigned int spin) { #if OWS_RANK_DIAG_FALLBACK static int warned = 0; if(ows_try_mark_once(&warned)) fprintf(stderr, "[OWS][rank] WARNING: rank profile init timeout (spin=%u), using fallback defaults\n", spin); #else (void)spin; #endif } static struct ows_rank_profiles_cfg ows_rank_profiles_fallback(void) { struct ows_rank_profiles_cfg fb; memset(&fb, 0, sizeof(fb)); fb.count = 1; snprintf(fb.profiles[0].name, sizeof(fb.profiles[0].name), "default"); ows_rank_weights_defaults(&fb.profiles[0].w); ows_rank_weights_clip(&fb.profiles[0].w); fb.loaded_from_file = 0; return fb; } static void ows_rank_weights_clip(struct ows_rank_weights_cfg *cfg) { double wSum; if(!cfg) return; cfg->w_level = ows_clip(cfg->w_level, 0.01, 5.0); cfg->w_title = ows_clip(cfg->w_title, 0.01, 5.0); cfg->w_text = ows_clip(cfg->w_text, 0.01, 5.0); wSum = cfg->w_level + cfg->w_title + cfg->w_text; if(wSum <= 0.0 || wSum != wSum) { cfg->w_level = 0.52; cfg->w_title = 0.20; cfg->w_text = 0.28; } else { cfg->w_level /= wSum; cfg->w_title /= wSum; cfg->w_text /= wSum; } cfg->std_level = ows_clip(cfg->std_level, 0.0, 0.5); cfg->std_title = ows_clip(cfg->std_title, 0.0, 0.5); cfg->std_text = ows_clip(cfg->std_text, 0.0, 0.5); ows_rank_limit_profile_amplitude(cfg->w_level, cfg->w_title, cfg->w_text, &cfg->std_level, &cfg->std_title, &cfg->std_text); } static char ows_ascii_lower(char c) { if(c >= 'A' && c <= 'Z') return (char)(c + ('a' - 'A')); return c; } static void ows_norm_profile_name(const char *in, char *out, size_t out_cap) { size_t i; size_t n; if(out==NULL || out_cap==0) return; out[0] = '\0'; if(in==NULL || in[0]=='\0') { snprintf(out, out_cap, "default"); return; } n = strlen(in); if(n >= out_cap) n = out_cap - 1; for(i = 0; i < n; i++) { char c = ows_ascii_lower(in[i]); if((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c=='_' || c=='-') out[i] = c; else out[i] = '_'; } out[n] = '\0'; if(out[0]=='\0') snprintf(out, out_cap, "default"); } static struct ows_rank_profile_cfg *ows_profiles_find(struct ows_rank_profiles_cfg *cfg, const char *name) { int i; if(cfg==NULL || name==NULL || name[0]=='\0') return NULL; for(i = 0; i < cfg->count; i++) { if(strcmp(cfg->profiles[i].name, name) == 0) return &cfg->profiles[i]; } return NULL; } static struct ows_rank_profile_cfg *ows_profiles_add(struct ows_rank_profiles_cfg *cfg, const char *name) { struct ows_rank_profile_cfg *p; if(cfg==NULL || name==NULL || name[0]=='\0') return NULL; p = ows_profiles_find(cfg, name); if(p) return p; if(cfg->count >= OWS_MAX_RANK_PROFILES) return NULL; p = &cfg->profiles[cfg->count++]; memset(p, 0, sizeof(*p)); ows_rank_copy_string(p->name, sizeof(p->name), name); ows_rank_weights_defaults(&p->w); ows_rank_weights_clip(&p->w); return p; } static struct ows_rank_profiles_cfg ows_load_rank_profiles_once(void) { /* initState: 0=not started, 1=initializing, 2=ready, 3=failed_fallback */ #if OWS_HAVE_ATOMICS static int initState = 0; #else static int isInit = 0; #endif static struct ows_rank_profiles_cfg cfg; struct ows_rank_profile_cfg *cur; FILE *fp; char line[256]; int i; #if OWS_HAVE_ATOMICS int expected; #endif #if OWS_HAVE_ATOMICS if(__atomic_load_n(&initState, __ATOMIC_ACQUIRE) == 2) return cfg; if(__atomic_load_n(&initState, __ATOMIC_ACQUIRE) == 3) return ows_rank_profiles_fallback(); expected = 0; if(!__atomic_compare_exchange_n(&initState, &expected, 1, 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) { unsigned int spin = 0; while(1) { int st = __atomic_load_n(&initState, __ATOMIC_ACQUIRE); if(st == 2) return cfg; if(st == 3) return ows_rank_profiles_fallback(); ows_cpu_relax_backoff(&spin); if(spin >= OWS_PROFILE_INIT_SPIN_MAX) { expected = 1; (void)__atomic_compare_exchange_n(&initState, &expected, 3, 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE); ows_rank_diag_fallback_once(spin); return ows_rank_profiles_fallback(); } } } #else if(isInit) return cfg; isInit = 1; #endif memset(&cfg, 0, sizeof(cfg)); cur = ows_profiles_add(&cfg, "default"); if(cur==NULL) { #if OWS_HAVE_ATOMICS __atomic_store_n(&initState, 2, __ATOMIC_RELEASE); #endif return cfg; } fp = ows_fopen_config("rank_weights.conf", "r", NULL, 0); if(!fp) { #if OWS_HAVE_ATOMICS __atomic_store_n(&initState, 2, __ATOMIC_RELEASE); #endif return cfg; } while(fgets(line, sizeof(line), fp)) { size_t ln; char *eq; char key[128]; char val[128]; char sec[OWS_RANK_PROFILE_NAME_MAX]; double d; ows_trim_inplace(line); if(line[0] == '\0' || line[0] == '#' || line[0] == ';') continue; ln = strlen(line); if(ln >= 3 && line[0] == '[' && line[ln-1] == ']') { line[ln-1] = '\0'; ows_trim_inplace(line + 1); ows_norm_profile_name(line + 1, sec, sizeof(sec)); cur = ows_profiles_add(&cfg, sec); if(cur == NULL) cur = ows_profiles_find(&cfg, "default"); continue; } eq = strchr(line, '='); if(!eq) continue; *eq = '\0'; if(!ows_rank_copy_string(key, sizeof(key), line)) continue; if(!ows_rank_copy_string(val, sizeof(val), eq + 1)) continue; ows_trim_inplace(key); ows_trim_inplace(val); if(!ows_rank_parse_double(val, 0.0, &d)) continue; if(cur == NULL) continue; if(strcmp(key, "w_level") == 0) cur->w.w_level = d; else if(strcmp(key, "w_title") == 0) cur->w.w_title = d; else if(strcmp(key, "w_text") == 0) cur->w.w_text = d; else if(strcmp(key, "std_level") == 0) cur->w.std_level = d; else if(strcmp(key, "std_title") == 0) cur->w.std_title = d; else if(strcmp(key, "std_text") == 0) cur->w.std_text = d; else continue; cur->w.loaded = 1; } fclose(fp); cfg.loaded_from_file = 1; for(i = 0; i < cfg.count; i++) ows_rank_weights_clip(&cfg.profiles[i].w); #if OWS_HAVE_ATOMICS __atomic_store_n(&initState, 2, __ATOMIC_RELEASE); #endif return cfg; } static void ows_rank_weights_blend(struct ows_rank_weights_cfg *out, const struct ows_rank_weights_cfg *a, const struct ows_rank_weights_cfg *b, double alpha) { double wSum; if(out==NULL || a==NULL || b==NULL) return; alpha = ows_clip(alpha, 0.0, 1.0); out->w_level = alpha * a->w_level + (1.0 - alpha) * b->w_level; out->w_title = alpha * a->w_title + (1.0 - alpha) * b->w_title; out->w_text = alpha * a->w_text + (1.0 - alpha) * b->w_text; wSum = out->w_level + out->w_title + out->w_text; if(wSum > 0.0) { out->w_level /= wSum; out->w_title /= wSum; out->w_text /= wSum; } out->std_level = alpha * a->std_level + (1.0 - alpha) * b->std_level; out->std_title = alpha * a->std_title + (1.0 - alpha) * b->std_title; out->std_text = alpha * a->std_text + (1.0 - alpha) * b->std_text; out->loaded = a->loaded || b->loaded; ows_rank_weights_clip(out); } static int ows_fetch_manual_host_profile(int host_id, char *profileOut, size_t profileCap, double *confidenceOut) { MYSQL_RES **tmpRes; MYSQL_RES gRes; MYSQL_RES *res; MYSQL_ROW row; char sqlQuery[MAXQUERYSIZE]; char norm[OWS_RANK_PROFILE_NAME_MAX]; OWS_RANK_GUARD_RET(ows_rank_guard_host_id(host_id), 0); OWS_RANK_GUARD_RET(ows_rank_guard_output(profileOut, profileCap), 0); profileOut[0] = '\0'; if(confidenceOut) *confidenceOut = 0.0; tmpRes=(MYSQL_RES **)calloc(1, sizeof(MYSQL_RES *)); if(tmpRes==NULL) return 0; if(!ows_rank_guard_sqlbuf(sqlQuery, sizeof(sqlQuery))) { FREE(tmpRes); return 0; } if(!ows_rank_snprintf_ok(snprintf(sqlQuery, sizeof(sqlQuery), "SELECT profile, confidence FROM host_intent WHERE host_id=%d LIMIT 1", host_id), sizeof(sqlQuery))) { FREE(tmpRes); return 0; } my_mysql_ping(&gMysqlDB2, BLOCKINDEX); if(my_mysql_query_and_store_results(&gMysqlDB2, sqlQuery, tmpRes, &gRes, BLOCKINDEX) != 0) { FREE(tmpRes); return 0; } res = ows_rank_guard_tmp_result(tmpRes); if(res==NULL || !ows_rank_guard_mysql_fields(res, 2U)) { if(res) mysql_free_result(res); FREE(tmpRes); return 0; } row = mysql_fetch_row(res); if(ows_rank_guard_row_text(row, 0U)) { ows_norm_profile_name(ows_rank_row_text_or_null(row, 0U), norm, sizeof(norm)); if(!ows_rank_copy_string(profileOut, profileCap, norm)) { mysql_free_result(res); FREE(tmpRes); return 0; } if(confidenceOut) { double parsedConf = 0.95; if(ows_rank_guard_row_text(row, 1U)) ows_rank_row_parse_double(row, 1U, 0.95, &parsedConf); *confidenceOut = ows_clip(parsedConf, 0.0, 1.0); } mysql_free_result(res); FREE(tmpRes); return 1; } mysql_free_result(res); FREE(tmpRes); return 0; } static int ows_guess_profile_for_host(int host_id, char *profileOut, size_t profileCap, double *confidenceOut) { MYSQL_RES **tmpRes; MYSQL_RES gRes; MYSQL_RES *res; MYSQL_ROW row; char sqlQuery[MAXQUERYSIZE]; double total; double pdfRatio; double ecomRatio; double docsRatio; double blogRatio; double scores[4]; const char *names[4]; int bestIdx; int i; double bestScore; double secondScore; double conf; OWS_RANK_GUARD_RET(ows_rank_guard_host_id(host_id), 0); OWS_RANK_GUARD_RET(ows_rank_guard_output(profileOut, profileCap), 0); profileOut[0] = '\0'; if(confidenceOut) *confidenceOut = 0.0; tmpRes=(MYSQL_RES **)calloc(1, sizeof(MYSQL_RES *)); if(tmpRes==NULL) return 0; if(!ows_rank_guard_sqlbuf(sqlQuery, sizeof(sqlQuery))) { FREE(tmpRes); return 0; } if(!ows_rank_snprintf_ok(snprintf(sqlQuery, sizeof(sqlQuery), "SELECT COUNT(*)," "SUM(CASE WHEN url LIKE '%%.pdf' OR url LIKE '%%.PDF' OR url LIKE '%%.pdf?%%' OR url LIKE '%%.PDF?%%' THEN 1 ELSE 0 END)," "SUM(CASE WHEN url LIKE '%%/product%%' OR url LIKE '%%/shop%%' OR url LIKE '%%/cart%%' OR url LIKE '%%/checkout%%' THEN 1 ELSE 0 END)," "SUM(CASE WHEN title LIKE '%%manual%%' OR title LIKE '%%Manual%%' OR title LIKE '%%MANUAL%%' " "OR title LIKE '%%documentation%%' OR title LIKE '%%Documentation%%' OR title LIKE '%%DOCUMENTATION%%' " "OR title LIKE '%%api%%' OR title LIKE '%%API%%' " "OR title LIKE '%%guide%%' OR title LIKE '%%Guide%%' OR title LIKE '%%GUIDE%%' THEN 1 ELSE 0 END)," "SUM(CASE WHEN url LIKE '%%/tag/%%' OR url LIKE '%%/category/%%' OR url LIKE '%%/20__/__/%%' OR url LIKE '%%/20__/_%%/%%' THEN 1 ELSE 0 END) " "FROM pagelist WHERE host_id=%d", host_id), sizeof(sqlQuery))) { FREE(tmpRes); return 0; } my_mysql_ping(&gMysqlDB2, BLOCKINDEX); if(my_mysql_query_and_store_results(&gMysqlDB2, sqlQuery, tmpRes, &gRes, BLOCKINDEX) != 0) { FREE(tmpRes); return 0; } res = ows_rank_guard_tmp_result(tmpRes); if(res==NULL || !ows_rank_guard_mysql_fields(res, 5U)) { if(res) mysql_free_result(res); FREE(tmpRes); return 0; } row = mysql_fetch_row(res); if(!ows_rank_guard_row(row)) { mysql_free_result(res); FREE(tmpRes); return 0; } ows_rank_row_parse_double(row, 0U, 0.0, &total); if(total <= 0.0) { mysql_free_result(res); FREE(tmpRes); return 0; } ows_rank_row_parse_double(row, 1U, 0.0, &pdfRatio); ows_rank_row_parse_double(row, 2U, 0.0, &ecomRatio); ows_rank_row_parse_double(row, 3U, 0.0, &docsRatio); ows_rank_row_parse_double(row, 4U, 0.0, &blogRatio); pdfRatio = ows_clip(pdfRatio / total, 0.0, 1.0); ecomRatio = ows_clip(ecomRatio / total, 0.0, 1.0); docsRatio = ows_clip(docsRatio / total, 0.0, 1.0); blogRatio = ows_clip(blogRatio / total, 0.0, 1.0); mysql_free_result(res); FREE(tmpRes); names[0] = "pdf_heavy"; names[1] = "docs"; names[2] = "ecommerce"; names[3] = "blog"; scores[0] = (1.6 * pdfRatio) + (0.3 * docsRatio); scores[1] = (1.2 * docsRatio) + (0.4 * pdfRatio); scores[2] = (1.4 * ecomRatio); scores[3] = (1.1 * blogRatio) + (0.2 * docsRatio); bestIdx = 0; bestScore = scores[0]; secondScore = 0.0; for(i = 1; i < 4; i++) { if(scores[i] > bestScore) { secondScore = bestScore; bestScore = scores[i]; bestIdx = i; } else if(scores[i] > secondScore) { secondScore = scores[i]; } } conf = ows_clip((bestScore - secondScore) * 2.0 + bestScore, 0.0, 1.0); if(conf < 0.25 || bestScore < 0.15) { if(!ows_rank_copy_string(profileOut, profileCap, "default")) return 0; if(confidenceOut) *confidenceOut = conf; return 1; } if(!ows_rank_copy_string(profileOut, profileCap, names[bestIdx])) return 0; if(confidenceOut) *confidenceOut = conf; return 1; } static void ows_guess_profile_for_url_flags(int isPdf, int isEcom, int isDocs, int isBlog, int isChaotic, int level, int titleLen, int textLen, char *profileOut, size_t profileCap, double *confidenceOut, double *orderOut, double *chaosOut) { const char *name = "default"; double presence = 0.0; double order = 0.0; double chaos = 0.0; double conf; if(isPdf) { name = "pdf_heavy"; presence = 1.0; } else if(isDocs) { name = "docs"; presence = 1.0; } else if(isEcom) { name = "ecommerce"; presence = 1.0; } else if(isBlog) { name = "blog"; presence = 1.0; } if(!isChaotic) order += 0.6; if(level <= 2) order += 0.4; order = ows_clip(order, 0.0, 1.0); if(isChaotic) chaos += 0.7; if(textLen <= 0) chaos += 0.3; if(titleLen <= 0) chaos += 0.2; chaos = ows_clip(chaos, 0.0, 1.0); conf = (0.50 * presence) + (0.25 * order) + (0.25 * (1.0 - chaos)); conf = ows_clip(conf, 0.0, 1.0); if(profileOut && profileCap > 0) ows_rank_copy_string(profileOut, profileCap, name); if(confidenceOut) *confidenceOut = conf; if(orderOut) *orderOut = order; if(chaosOut) *chaosOut = chaos; } static int ows_fetch_cached_host_profile(int host_id, char *profileOut, size_t profileCap, double *confidenceOut) { MYSQL_RES **tmpRes; MYSQL_RES gRes; MYSQL_RES *res; MYSQL_ROW row; char sqlQuery[MAXQUERYSIZE]; char norm[OWS_RANK_PROFILE_NAME_MAX]; OWS_RANK_GUARD_RET(ows_rank_guard_host_id(host_id), 0); OWS_RANK_GUARD_RET(ows_rank_guard_output(profileOut, profileCap), 0); profileOut[0] = '\0'; if(confidenceOut) *confidenceOut = 0.0; tmpRes=(MYSQL_RES **)calloc(1, sizeof(MYSQL_RES *)); if(tmpRes==NULL) return 0; if(!ows_rank_guard_sqlbuf(sqlQuery, sizeof(sqlQuery))) { FREE(tmpRes); return 0; } if(!ows_rank_snprintf_ok(snprintf(sqlQuery, sizeof(sqlQuery), "SELECT profile, confidence FROM host_intent_cache " "WHERE host_id=%d AND updated_at >= DATE_SUB(NOW(), INTERVAL %d DAY) LIMIT 1", host_id, OWS_INTENT_CACHE_TTL_DAYS), sizeof(sqlQuery))) { FREE(tmpRes); return 0; } my_mysql_ping(&gMysqlDB2, BLOCKINDEX); if(my_mysql_query_and_store_results(&gMysqlDB2, sqlQuery, tmpRes, &gRes, BLOCKINDEX) != 0) { FREE(tmpRes); return 0; } res = ows_rank_guard_tmp_result(tmpRes); if(res==NULL || !ows_rank_guard_mysql_fields(res, 2U)) { if(res) mysql_free_result(res); FREE(tmpRes); return 0; } row = mysql_fetch_row(res); if(ows_rank_guard_row_text(row, 0U)) { ows_norm_profile_name(ows_rank_row_text_or_null(row, 0U), norm, sizeof(norm)); if(!ows_rank_copy_string(profileOut, profileCap, norm)) { mysql_free_result(res); FREE(tmpRes); return 0; } if(confidenceOut) { double parsedConf = 0.0; if(ows_rank_guard_row_text(row, 1U)) ows_rank_row_parse_double(row, 1U, 0.0, &parsedConf); *confidenceOut = ows_clip(parsedConf, 0.0, 1.0); } mysql_free_result(res); FREE(tmpRes); return 1; } mysql_free_result(res); FREE(tmpRes); return 0; } static void ows_store_cached_host_profile(int host_id, const char *profile, double confidence) { char sqlQuery[MAXQUERYSIZE]; char profileEsc[(OWS_RANK_PROFILE_NAME_MAX * 2) + 1]; char profileNorm[OWS_RANK_PROFILE_NAME_MAX]; static int _tblChecked = 0; OWS_RANK_GUARD_VOID(ows_rank_guard_host_id(host_id)); OWS_RANK_GUARD_VOID(ows_rank_guard_profile_name(profile)); ows_norm_profile_name(profile, profileNorm, sizeof(profileNorm)); my_mysql_ping(&gMysqlDB2, BLOCKINDEX); if(!ows_rank_escape_string(&gMysqlDB2, profileEsc, sizeof(profileEsc), profileNorm)) return; confidence = ows_clip(confidence, 0.0, 1.0); if(ows_try_mark_once(&_tblChecked)) { my_mysql_ping(&gMysqlDB2, BLOCKINDEX); if(!ows_rank_snprintf_ok(snprintf(sqlQuery, sizeof(sqlQuery), "CREATE TABLE IF NOT EXISTS host_intent_cache (" "host_id INT(10) UNSIGNED NOT NULL PRIMARY KEY," "profile VARCHAR(32) NOT NULL," "confidence DOUBLE NOT NULL DEFAULT 0," "updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP) " "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"), sizeof(sqlQuery))) return; my_mysql_query(&gMysqlDB2, sqlQuery, NO_BLOCK); } if(!ows_rank_snprintf_ok(snprintf(sqlQuery, sizeof(sqlQuery), "REPLACE INTO host_intent_cache(host_id,profile,confidence,updated_at) VALUES(%d,'%s',%.6f,NOW())", host_id, profileEsc, confidence), sizeof(sqlQuery))) return; my_mysql_ping(&gMysqlDB2, BLOCKINDEX); my_mysql_query(&gMysqlDB2, sqlQuery, NO_BLOCK); } static void ows_ensure_url_intent_cache_table(void) { static int _tblChecked = 0; char sqlQuery[MAXQUERYSIZE]; if(!ows_try_mark_once(&_tblChecked)) return; my_mysql_ping(&gMysqlDB2, BLOCKINDEX); OWS_RANK_GUARD_SQL_VOID(snprintf(sqlQuery, sizeof(sqlQuery), "CREATE TABLE IF NOT EXISTS url_intent_cache (" "host_id INT(10) UNSIGNED NOT NULL," "page_id INT(10) UNSIGNED NOT NULL," "profile VARCHAR(32) NOT NULL," "confidence DOUBLE NOT NULL DEFAULT 0," "order_score DOUBLE NOT NULL DEFAULT 0," "chaos_score DOUBLE NOT NULL DEFAULT 0," "updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," "PRIMARY KEY(host_id,page_id)," "KEY idx_updated (updated_at)) " "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"), sizeof(sqlQuery)); my_mysql_query(&gMysqlDB2, sqlQuery, NO_BLOCK); } static void ows_url_cache_flush(char *sqlQuery, char *valBuf, int *count, long *flushes, long *rows, int *minRows, int *maxRows, int flushReason, long *flushByCount, long *flushBy75, long *flushByCap, long *anomalyFlushSkipped, long *flushFinal, long *rowsFinal) { int batchRows = (count ? *count : 0); if(!ows_rank_guard_sqlbuf(sqlQuery, (size_t)MAXQUERYSIZE) || !ows_rank_guard_batch_buffer(valBuf, (size_t)MAXQUERYSIZE)) { if(batchRows > 0) { if(anomalyFlushSkipped) (*anomalyFlushSkipped)++; if(count) *count = 0; } return; } if(valBuf[0]=='\0') { if(batchRows > 0) { if(anomalyFlushSkipped) (*anomalyFlushSkipped)++; if(count) *count = 0; } return; } if(batchRows > 0) { if(flushes) (*flushes)++; if(rows) (*rows) += (long)batchRows; if(minRows && (*minRows == 0 || batchRows < *minRows)) *minRows = batchRows; if(maxRows && batchRows > *maxRows) *maxRows = batchRows; if(flushReason == OWS_URL_CACHE_FLUSH_BY_COUNT && flushByCount) (*flushByCount)++; else if(flushReason == OWS_URL_CACHE_FLUSH_BY_75 && flushBy75) (*flushBy75)++; else if(flushReason == OWS_URL_CACHE_FLUSH_BY_CAP && flushByCap) (*flushByCap)++; else if(flushReason == OWS_URL_CACHE_FLUSH_FINAL) { if(flushFinal) (*flushFinal)++; if(rowsFinal) (*rowsFinal) += (long)batchRows; } } my_mysql_ping(&gMysqlDB2, BLOCKINDEX); if(!ows_rank_snprintf_ok(snprintf(sqlQuery, MAXQUERYSIZE, "REPLACE INTO url_intent_cache(host_id,page_id,profile,confidence,order_score,chaos_score,updated_at) " "VALUES %s", valBuf), (size_t)MAXQUERYSIZE)) { if(anomalyFlushSkipped) (*anomalyFlushSkipped)++; ows_rank_reset_buffer(valBuf, (size_t)MAXQUERYSIZE); if(count) *count = 0; return; } my_mysql_query(&gMysqlDB2, sqlQuery, NO_BLOCK); ows_rank_reset_buffer(valBuf, (size_t)MAXQUERYSIZE); if(count) *count = 0; } static int ows_url_cache_append(int host_id, int page_id, const char *profile, double conf, double order, double chaos, char *sqlQuery, char *valBuf, size_t cap, int *count, long *flushes, long *rows, int *minRows, int *maxRows, long *flushByCount, long *flushBy75, long *flushByCap, long *anomalyFlushSkipped, long *skipInvalid, long *skipFormat, long *skipNoSpace) { char profEsc[(OWS_RANK_PROFILE_NAME_MAX * 2) + 1]; char profNorm[OWS_RANK_PROFILE_NAME_MAX]; char rowTmp[256]; size_t len; int wr; int flushReason = OWS_URL_CACHE_FLUSH_FINAL; if(!ows_rank_guard_sqlbuf(sqlQuery, (size_t)MAXQUERYSIZE) || !ows_rank_guard_batch_buffer(valBuf, cap) || !ows_rank_guard_batch_count(count) || !ows_rank_guard_host_id(host_id) || !ows_rank_guard_positive_int(page_id)) { if(skipInvalid) (*skipInvalid)++; return 0; } profile = ows_rank_string_or_default(profile, "default"); ows_norm_profile_name(profile, profNorm, sizeof(profNorm)); if(*count == 0) my_mysql_ping(&gMysqlDB2, BLOCKINDEX); conf = ows_clip(conf, 0.0, 1.0); order = ows_clip(order, 0.0, 1.0); chaos = ows_clip(chaos, 0.0, 1.0); if(!ows_rank_escape_string(&gMysqlDB2, profEsc, sizeof(profEsc), profNorm)) { if(skipFormat) (*skipFormat)++; return 0; } len = strlen(valBuf); wr = snprintf(rowTmp, sizeof(rowTmp), "%s(%d,%d,'%s',%.6f,%.6f,%.6f,NOW())", (len > 0 ? "," : ""), host_id, page_id, profEsc, conf, order, chaos); if(!ows_rank_guard_sql_piece(wr, sizeof(rowTmp))) { if(skipFormat) (*skipFormat)++; return 0; } if((*count) >= OWS_URL_CACHE_BATCH_MAX) flushReason = OWS_URL_CACHE_FLUSH_BY_COUNT; else if(len >= ((cap * 3U) / 4U)) flushReason = OWS_URL_CACHE_FLUSH_BY_75; else if((len + (size_t)wr + 8U) >= cap) flushReason = OWS_URL_CACHE_FLUSH_BY_CAP; if(flushReason != OWS_URL_CACHE_FLUSH_FINAL) { ows_url_cache_flush(sqlQuery, valBuf, count, flushes, rows, minRows, maxRows, flushReason, flushByCount, flushBy75, flushByCap, anomalyFlushSkipped, NULL, NULL); wr = snprintf(rowTmp, sizeof(rowTmp), "(%d,%d,'%s',%.6f,%.6f,%.6f,NOW())", host_id, page_id, profEsc, conf, order, chaos); if(!ows_rank_guard_sql_piece(wr, sizeof(rowTmp))) { if(skipFormat) (*skipFormat)++; return 0; } len = 0; } if((len + (size_t)wr + 1U) >= cap) { if(skipNoSpace) (*skipNoSpace)++; return 0; } memcpy(valBuf + len, rowTmp, (size_t)wr + 1U); (*count)++; return 1; } static struct ows_rank_profile_pick ows_pick_profile_for_host(int host_id) { struct ows_rank_profile_pick pick; struct ows_rank_profiles_cfg cfg; struct ows_rank_profile_cfg *defp; struct ows_rank_profile_cfg *chosen; char name[OWS_RANK_PROFILE_NAME_MAX]; double conf; int found; memset(&pick, 0, sizeof(pick)); cfg = ows_load_rank_profiles_once(); defp = ows_profiles_find(&cfg, "default"); if(defp == NULL && cfg.count > 0) defp = &cfg.profiles[0]; if(defp == NULL) { ows_rank_weights_defaults(&pick.w); ows_rank_copy_string(pick.name, sizeof(pick.name), "default"); ows_rank_copy_string(pick.source, sizeof(pick.source), "hardcoded"); pick.confidence = 0.0; return pick; } pick.w = defp->w; ows_rank_copy_string(pick.name, sizeof(pick.name), "default"); ows_rank_copy_string(pick.source, sizeof(pick.source), defp->w.loaded ? "conf:default" : "hardcoded"); pick.confidence = 1.0; pick.conf_file_loaded = cfg.loaded_from_file; ows_rank_reset_buffer(name, sizeof(name)); conf = 0.0; found = ows_fetch_manual_host_profile(host_id, name, sizeof(name), &conf); if(found) ows_rank_copy_string(pick.source, sizeof(pick.source), "manual"); else if(ows_fetch_cached_host_profile(host_id, name, sizeof(name), &conf)) ows_rank_copy_string(pick.source, sizeof(pick.source), "heuristic_cached"); else if(ows_guess_profile_for_host(host_id, name, sizeof(name), &conf)) { ows_rank_copy_string(pick.source, sizeof(pick.source), "heuristic_fresh"); ows_store_cached_host_profile(host_id, name, conf); } else return pick; chosen = ows_profiles_find(&cfg, name); if(chosen == NULL && strcmp(name, "default") != 0) { chosen = ows_profiles_find(&cfg, "default"); if(strcmp(pick.source, "manual") == 0) ows_rank_copy_string(pick.source, sizeof(pick.source), "manual->default"); else if(strcmp(pick.source, "heuristic_cached") == 0) ows_rank_copy_string(pick.source, sizeof(pick.source), "heuristic_cached->default"); else if(strcmp(pick.source, "heuristic_fresh") == 0) ows_rank_copy_string(pick.source, sizeof(pick.source), "heuristic_fresh->default"); } if(chosen == NULL) { ows_rank_copy_string(pick.source, sizeof(pick.source), "fallback:default"); return pick; } if(strcmp(chosen->name, "default") == 0) { pick.confidence = ows_clip(conf, 0.0, 1.0); ows_rank_copy_string(pick.name, sizeof(pick.name), "default"); if(strcmp(pick.source, "manual") == 0) ows_rank_copy_string(pick.source, sizeof(pick.source), "manual->default"); else if(strcmp(pick.source, "heuristic_cached") == 0) ows_rank_copy_string(pick.source, sizeof(pick.source), "heuristic_cached->default"); else if(strcmp(pick.source, "heuristic_fresh") == 0) ows_rank_copy_string(pick.source, sizeof(pick.source), "heuristic_fresh->default"); return pick; } if(conf < 0.5) { pick.confidence = conf; if(strcmp(pick.source, "manual") == 0) ows_rank_copy_string(pick.source, sizeof(pick.source), "manual->default"); else if(strcmp(pick.source, "heuristic_cached") == 0) ows_rank_copy_string(pick.source, sizeof(pick.source), "heuristic_cached->default"); else if(strcmp(pick.source, "heuristic_fresh") == 0) ows_rank_copy_string(pick.source, sizeof(pick.source), "heuristic_fresh->default"); return pick; } pick.confidence = ows_clip(conf, 0.0, 1.0); ows_rank_weights_blend(&pick.w, &chosen->w, &defp->w, pick.confidence); ows_rank_copy_string(pick.name, sizeof(pick.name), chosen->name); if(strcmp(pick.source, "manual") == 0) (void)ows_rank_guard_sql_piece(snprintf(pick.source, sizeof(pick.source), "manual:%s", chosen->name), sizeof(pick.source)); else if(strcmp(pick.source, "heuristic_cached") == 0) (void)ows_rank_guard_sql_piece(snprintf(pick.source, sizeof(pick.source), "heuristic_cached:%s", chosen->name), sizeof(pick.source)); else if(strcmp(pick.source, "heuristic_fresh") == 0) (void)ows_rank_guard_sql_piece(snprintf(pick.source, sizeof(pick.source), "heuristic_fresh:%s", chosen->name), sizeof(pick.source)); return pick; } static double ows_clamp01(double v) { if(v != v) return 0.0; if(v < 0.0) return 0.0; if(v > 1.0) return 1.0; return v; } static double ows_rank_mc_trimmed_mean(double sum, double minv, double maxv, int rounds) { if(rounds <= 0) return 0.0; if(rounds >= 5 && minv <= maxv) return ows_clamp01((sum - minv - maxv) / (double)(rounds - 2)); return ows_clamp01(sum / (double)rounds); } static double ows_rank_mc_risk_adjusted_score(double robust, double mcStd) { double penalty; if(!ows_rank_guard_unit_interval(robust)) robust = ows_clamp01(robust); if(ows_rank_bad_number(mcStd) || mcStd < 0.0) mcStd = 0.0; penalty = ows_clip(mcStd * OWS_RANK_MC_STD_PENALTY, 0.0, OWS_RANK_MC_STD_PENALTY_MAX); return ows_clamp01(robust - penalty); } static int ows_rank_mc_value_ok(double v) { return !ows_rank_bad_number(v) && v >= 0.0 && v <= 1.0; } static double ows_rank_deterministic_score(double level, double title, double text, double wLevel, double wTitle, double wText, double order, double chaos) { double wSum; double margin; level = ows_clamp01(level); title = ows_clamp01(title); text = ows_clamp01(text); wLevel = ows_clip(wLevel, 0.01, 1.0); wTitle = ows_clip(wTitle, 0.01, 1.0); wText = ows_clip(wText, 0.01, 1.0); order = ows_clamp01(order); chaos = ows_clamp01(chaos); wSum = wLevel + wTitle + wText; if(ows_rank_bad_number(wSum) || wSum <= 0.0) { wLevel = 0.3333333333333333; wTitle = 0.3333333333333333; wText = 0.3333333333333333; } else { wLevel /= wSum; wTitle /= wSum; wText /= wSum; } margin = (wLevel * level) + (wTitle * title) + (wText * text); margin += (0.05 * order) - (0.05 * chaos); return ows_clamp01(margin); } static double ows_sqrt_fast(double x) { double g; int i; if(x <= 0.0) return 0.0; g = (x > 1.0) ? x : 1.0; for(i = 0; i < 8; i++) g = 0.5 * (g + (x / g)); return g; } /* Fast log(1+n) approximation without libm dependency. */ static double ows_log1p_fast_int(int n) { double x; double y; double z; double z2; double t; double s; int k; if(n <= 0) return 0.0; if(n > 10000000) n = 10000000; x = (double)n + 1.0; y = 0.0; while(x > 1.5) { x *= 0.5; y += 0.6931471805599453; /* ln(2) */ } while(x < 0.75) { x *= 2.0; y -= 0.6931471805599453; /* ln(2) */ } z = (x - 1.0) / (x + 1.0); z2 = z * z; t = z; s = 0.0; for(k = 1; k <= 9; k += 2) { s += t / (double)k; t *= z2; } return y + (2.0 * s); } static unsigned int ows_xorshift32(unsigned int *state) { unsigned int x = (*state == 0U) ? 2463534242U : *state; x ^= x << 13; x ^= x >> 17; x ^= x << 5; *state = x; return x; } static double ows_rand_sym_local(unsigned int *state, double amp) { double u; if(amp <= 0.0) return 0.0; u = ((double)ows_xorshift32(state) / 4294967295.0) * 2.0 - 1.0; return u * amp; } static double ows_vdc_unit(unsigned int n, unsigned int base) { double denom = 1.0; double out = 0.0; if(base < 2U) base = 2U; while(n > 0U) { denom *= (double)base; out += (double)(n % base) / denom; n /= base; } return ows_clamp01(out); } static double ows_lowdisc_sym(unsigned int n, unsigned int base, double amp) { if(amp <= 0.0) return 0.0; return ((ows_vdc_unit(n, base) * 2.0) - 1.0) * amp; } static void ows_flush_rank_case_batch(int host_id, char *sqlQuery, char *caseBuf, char *idBuf) { if(!ows_rank_guard_host_id(host_id) || !ows_rank_guard_sqlbuf(sqlQuery, (size_t)MAXQUERYSIZE) || !ows_rank_guard_nonempty_string(caseBuf) || !ows_rank_guard_nonempty_string(idBuf)) return; if(!ows_rank_snprintf_ok(snprintf(sqlQuery, MAXQUERYSIZE, "UPDATE pagelist SET rank = CASE id %sELSE rank END WHERE host_id=%d AND id IN (%s)", caseBuf, host_id, idBuf), (size_t)MAXQUERYSIZE)) return; my_mysql_query(&gMysqlDB2, sqlQuery, NO_BLOCK); } static int ows_buf_append(char *buf, size_t *len, size_t cap, const char *txt) { int wr; size_t rem; if(!ows_rank_guard_buf_append_args(buf, len, cap, txt)) return 0; if(*len >= (cap - 1)) return 0; rem = cap - *len; wr = snprintf(buf + *len, rem, "%s", txt); if(wr < 0) return 0; if((size_t)wr >= rem) return 0; /* truncated */ *len += (size_t)wr; return 1; } static int ows_try_append_rank_row( int host_id, char *sqlQuery, char *caseBuf, size_t *lenCase, char *idBuf, size_t *lenId, int *batchCount, const char *rowCase, const char *rowId) { if(!ows_rank_guard_rank_batch_args(sqlQuery, caseBuf, lenCase, idBuf, lenId, batchCount, rowCase, rowId)) return 0; if(*batchCount >= 120 || *lenCase >= (size_t)(MAXQUERYSIZE - 1) || *lenId >= (size_t)(MAXQUERYSIZE - 1) || ((size_t)MAXQUERYSIZE - *lenCase) < 80 || ((size_t)MAXQUERYSIZE - *lenId) < 32) { ows_flush_rank_case_batch(host_id, sqlQuery, caseBuf, idBuf); ows_rank_reset_buffer(caseBuf, (size_t)MAXQUERYSIZE); ows_rank_reset_buffer(idBuf, (size_t)MAXQUERYSIZE); *lenCase = 0; *lenId = 0; *batchCount = 0; } if(!ows_buf_append(caseBuf, lenCase, (size_t)MAXQUERYSIZE, rowCase)) goto retry_flush; if(*batchCount > 0) { if(!ows_buf_append(idBuf, lenId, (size_t)MAXQUERYSIZE, ",")) goto retry_flush; } if(!ows_buf_append(idBuf, lenId, (size_t)MAXQUERYSIZE, rowId)) goto retry_flush; (*batchCount)++; return 1; retry_flush: ows_flush_rank_case_batch(host_id, sqlQuery, caseBuf, idBuf); ows_rank_reset_buffer(caseBuf, (size_t)MAXQUERYSIZE); ows_rank_reset_buffer(idBuf, (size_t)MAXQUERYSIZE); *lenCase = 0; *lenId = 0; *batchCount = 0; if(!ows_buf_append(caseBuf, lenCase, (size_t)MAXQUERYSIZE, rowCase)) return 0; if(!ows_buf_append(idBuf, lenId, (size_t)MAXQUERYSIZE, rowId)) return 0; (*batchCount)++; return 1; } static void ows_save_rank_stability_host(int host_id, double stability, double avgStd, double maxStd, int rounds, double meanScore, double varScore, const char *profile, const char *selector, double confidence) { char sqlQuery[MAXQUERYSIZE]; char profileEsc[(OWS_RANK_PROFILE_NAME_MAX * 2) + 1]; char selectorEsc[(64 * 2) + 1]; char profileNorm[OWS_RANK_PROFILE_NAME_MAX]; char selectorNorm[64]; static int _tblChecked = 0; OWS_RANK_GUARD_VOID(ows_rank_guard_host_id(host_id)); profile = ows_rank_string_or_default(profile, ""); selector = ows_rank_string_or_default(selector, ""); stability = ows_clip(stability, 0.0, 100.0); avgStd = ows_clip(avgStd, 0.0, 1.0); maxStd = ows_clip(maxStd, 0.0, 1.0); meanScore = ows_clip(meanScore, 0.0, 1.0); varScore = ows_clip(varScore, 0.0, 1.0); if(rounds < 0) rounds = 0; if(ows_try_mark_once(&_tblChecked)) { int ret; my_mysql_ping(&gMysqlDB2, BLOCKINDEX); if(!ows_rank_snprintf_ok(snprintf(sqlQuery, sizeof(sqlQuery), "CREATE TABLE IF NOT EXISTS host_rank_metrics (" "host_id INT(10) UNSIGNED NOT NULL PRIMARY KEY," "stability DOUBLE NOT NULL DEFAULT 0," "avg_std DOUBLE NOT NULL DEFAULT 0," "max_std DOUBLE NOT NULL DEFAULT 0," "mean_score DOUBLE NOT NULL DEFAULT 0," "var_score DOUBLE NOT NULL DEFAULT 0," "profile VARCHAR(32) NOT NULL DEFAULT ''," "selector VARCHAR(64) NOT NULL DEFAULT ''," "confidence DOUBLE NOT NULL DEFAULT 0," "rounds INT(11) UNSIGNED NOT NULL DEFAULT 0," "updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP) " "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"), sizeof(sqlQuery))) return; my_mysql_query(&gMysqlDB2, sqlQuery, NO_BLOCK); ret = my_mysql_query(&gMysqlDB2, "ALTER TABLE host_rank_metrics ADD COLUMN profile VARCHAR(32) NOT NULL DEFAULT ''", NO_BLOCK); if(ret) { unsigned int errNo = mysql_errno(&gMysqlDB2); if(errNo != 1060U && errNo != 1061U) { /* 1060 duplicate column, 1061 duplicate key */ /* ignore non-fatal schema migration errors in runtime path */ } } ret = my_mysql_query(&gMysqlDB2, "ALTER TABLE host_rank_metrics ADD COLUMN selector VARCHAR(64) NOT NULL DEFAULT ''", NO_BLOCK); if(ret) { unsigned int errNo = mysql_errno(&gMysqlDB2); if(errNo != 1060U && errNo != 1061U) { } } ret = my_mysql_query(&gMysqlDB2, "ALTER TABLE host_rank_metrics ADD COLUMN confidence DOUBLE NOT NULL DEFAULT 0", NO_BLOCK); if(ret) { unsigned int errNo = mysql_errno(&gMysqlDB2); if(errNo != 1060U && errNo != 1061U) { } } } confidence = ows_clip(confidence, 0.0, 1.0); my_mysql_ping(&gMysqlDB2, BLOCKINDEX); ows_norm_profile_name(profile, profileNorm, sizeof(profileNorm)); if(!ows_rank_copy_string(selectorNorm, sizeof(selectorNorm), selector)) return; if(!ows_rank_escape_string(&gMysqlDB2, profileEsc, sizeof(profileEsc), profileNorm)) return; if(!ows_rank_escape_string(&gMysqlDB2, selectorEsc, sizeof(selectorEsc), selectorNorm)) return; if(!ows_rank_snprintf_ok(snprintf(sqlQuery, sizeof(sqlQuery), "REPLACE INTO host_rank_metrics(host_id,stability,avg_std,max_std,mean_score,var_score,profile,selector,confidence,rounds,updated_at)" " VALUES(%d,%.6f,%.6f,%.6f,%.6f,%.6f,'%s','%s',%.6f,%d,NOW())", host_id, stability, avgStd, maxStd, meanScore, varScore, profileEsc, selectorEsc, confidence, rounds), sizeof(sqlQuery))) return; my_mysql_query(&gMysqlDB2, sqlQuery, NO_BLOCK); } static int CalcPageRankLegacy(struct sHost host, int hostrank) { char sqlQuery[MAXQUERYSIZE]; /* Legacy formula: PR(px)=HostRank+(MaxPRLev/Level) */ if(!ows_rank_guard_host_id(host.host_id)) return 0; OWS_RANK_GUARD_SQL_RET(snprintf(sqlQuery, sizeof(sqlQuery), "UPDATE pagelist SET rank=%i+abs(%i/level) WHERE host_id = %d", ows_rank_safe_page_rank(hostrank, 0.0, 0), MAXPRLEV, host.host_id), sizeof(sqlQuery), 0); my_mysql_query(&gMysqlDB2, sqlQuery,NO_BLOCK); return 1; } int GetHostRank(int host_id) { MYSQL_RES **tmpRes; char sqlQuery[MAXQUERYSIZE]; MYSQL_RES gRes; MYSQL_ROW row; MYSQL_RES *res; int rank; OWS_RANK_GUARD_RET(ows_rank_guard_host_id(host_id), 0); tmpRes=(MYSQL_RES **)calloc(1, sizeof(MYSQL_RES *)); if(tmpRes==NULL) { MemoryCorruptedHandler("GetHostRank"); return 0; } if(!ows_rank_snprintf_ok(snprintf(sqlQuery, sizeof(sqlQuery), "select count(distinct host_id,linkedhost_id) from rels where linkedhost_id = %d", host_id), sizeof(sqlQuery))) { FREE(tmpRes); return 0; } my_mysql_ping(&gMysqlDB1, BLOCKDB1); if(!my_mysql_query_and_store_results(&gMysqlDB1, sqlQuery,tmpRes,&gRes,BLOCKDB1)) { res = ows_rank_guard_tmp_result(tmpRes); if(res==NULL || !ows_rank_guard_mysql_fields(res, 1U)) { if(res) mysql_free_result(res); FREE(tmpRes); return 0; } row = mysql_fetch_row(res); if(!ows_rank_guard_row_text(row, 0U)) { mysql_free_result(res); FREE(tmpRes); return 0; } ows_rank_row_parse_int(row, 0U, 0, &rank); mysql_free_result(res); FREE(tmpRes); return rank; } else { FREE(tmpRes); return 0; } } int CalcPageRank(struct sHost host) { char sqlQuery[MAXQUERYSIZE]; int hostrank; MYSQL_RES **tmpRes; MYSQL_RES gRes; MYSQL_RES *res; MYSQL_ROW row; long nPages; long i; long idx; int maxLevel; int maxTitleLen; int maxTextLen; int *ranks; int *ids; int *levels; int *titleLens; int *textLens; double *pageWLevel; double *pageWTitle; double *pageWText; double *pageStdLevel; double *pageStdTitle; double *pageStdText; double *pageOrder; double *pageChaos; double *accScores; double *accSqScores; double *minScores; double *maxScores; int *mcCounts; double meanScore; double varScore; double stability; double avgStd; double avgStdErr; double maxStd; double maxTitleLog; double maxTextLog; int rounds; unsigned int rngState; unsigned int rngW; unsigned int rngF; struct ows_rank_profile_pick pickedProfile; struct ows_rank_profiles_cfg profilesCfg; struct ows_rank_profile_cfg *defaultProfile; struct ows_rank_profile_cfg defaultFallback; double baseWLevel; double baseWTitle; double baseWText; double ampWLevel; double ampWTitle; double ampWText; char caseBuf[MAXQUERYSIZE]; char idBuf[MAXQUERYSIZE]; char cacheValBuf[MAXQUERYSIZE]; char pageProfileName[OWS_RANK_PROFILE_NAME_MAX]; int batchCount; int cacheBatchCount; long cacheBatchFlushes; long cacheRowsBatched; int cacheBatchMin; int cacheBatchMax; long cacheFlushByCount; long cacheFlushBy75; long cacheFlushByCap; long cacheFlushAnomaly; long cacheFlushFinal; long cacheRowsFinal; long cacheAppendOk; long cacheAppendSkip; long cacheAppendAttempted; long skipCauses; long cacheAppendSkipInvalid; long cacheAppendSkipFormat; long cacheAppendSkipNoSpace; size_t lenCase; size_t lenId; long cacheHit; long cacheMiss; long cacheMissForced; double stabilityScale; double stabilityRatio; double attemptedPerMiss; long mcAcceptedSamples; long mcRejectedSamples; long mcFallbackPages; double mcAcceptanceRate; double avgAcceptedRounds; #if OWS_RANK_DIAG_DUP_PID long dupPidMod; unsigned long long pidSeenMod[64]; /* 4096-bit modulo set */ #endif ids = NULL; ranks = NULL; levels = NULL; titleLens = NULL; textLens = NULL; pageWLevel = NULL; pageWTitle = NULL; pageWText = NULL; pageStdLevel = NULL; pageStdTitle = NULL; pageStdText = NULL; pageOrder = NULL; pageChaos = NULL; accScores = NULL; accSqScores = NULL; minScores = NULL; maxScores = NULL; mcCounts = NULL; /* In alcuni flussi host.host_id puo' arrivare a 0: proviamo a risolverlo dal DB prima di calcolare HostRank. */ if(host.host_id==0) host.host_id = GetHostId(host); OWS_RANK_GUARD_RET(ows_rank_guard_host_id(host.host_id), 0); hostrank=GetHostRank( host.host_id ); printf("\r\n + HostRank: %i",hostrank); printf("\r\n - Calculating Page Rank..."); profilesCfg = ows_load_rank_profiles_once(); defaultProfile = ows_profiles_find(&profilesCfg, "default"); if(defaultProfile == NULL && profilesCfg.count > 0) defaultProfile = &profilesCfg.profiles[0]; if(defaultProfile == NULL) { memset(&defaultFallback, 0, sizeof(defaultFallback)); snprintf(defaultFallback.name, sizeof(defaultFallback.name), "default"); ows_rank_weights_defaults(&defaultFallback.w); ows_rank_weights_clip(&defaultFallback.w); defaultProfile = &defaultFallback; } ows_ensure_url_intent_cache_table(); ows_rank_reset_buffer(cacheValBuf, (size_t)MAXQUERYSIZE); cacheBatchCount = 0; cacheBatchFlushes = 0; cacheRowsBatched = 0; cacheBatchMin = 0; cacheBatchMax = 0; cacheFlushByCount = 0; cacheFlushBy75 = 0; cacheFlushByCap = 0; cacheFlushAnomaly = 0; cacheFlushFinal = 0; cacheRowsFinal = 0; cacheAppendOk = 0; cacheAppendSkip = 0; cacheAppendAttempted = 0; skipCauses = 0; cacheAppendSkipInvalid = 0; cacheAppendSkipFormat = 0; cacheAppendSkipNoSpace = 0; cacheHit = 0; cacheMiss = 0; cacheMissForced = 0; attemptedPerMiss = 0.0; mcAcceptedSamples = 0; mcRejectedSamples = 0; mcFallbackPages = 0; mcAcceptanceRate = 0.0; avgAcceptedRounds = 0.0; #if OWS_RANK_DIAG_DUP_PID dupPidMod = 0; memset(pidSeenMod, 0, sizeof(pidSeenMod)); #endif /* 1) Read lightweight page vectors for this host. */ tmpRes=(MYSQL_RES **)calloc(1, sizeof(MYSQL_RES *)); if(tmpRes==NULL) { MemoryCorruptedHandler("CalcPageRank"); return CalcPageRankLegacy(host, hostrank); } if(!ows_rank_snprintf_ok(snprintf(sqlQuery, sizeof(sqlQuery), "SELECT p.id, p.level, CHAR_LENGTH(p.title), LENGTH(p.`text`), " "(p.page LIKE '%%.pdf' OR p.page LIKE '%%.PDF' OR p.page LIKE '%%.pdf?%%' OR p.page LIKE '%%.PDF?%%') AS is_pdf, " "(p.page LIKE '%%/product%%' OR p.page LIKE '%%/shop%%' OR p.page LIKE '%%/cart%%' OR p.page LIKE '%%/checkout%%') AS is_ecom, " "(p.title LIKE '%%manual%%' OR p.title LIKE '%%Manual%%' OR p.title LIKE '%%MANUAL%%' " "OR p.title LIKE '%%documentation%%' OR p.title LIKE '%%Documentation%%' OR p.title LIKE '%%DOCUMENTATION%%' " "OR p.title LIKE '%%api%%' OR p.title LIKE '%%API%%' " "OR p.title LIKE '%%guide%%' OR p.title LIKE '%%Guide%%' OR p.title LIKE '%%GUIDE%%') AS is_docs, " "(p.page LIKE '%%/tag/%%' OR p.page LIKE '%%/category/%%' OR p.page LIKE '%%/20__/__/%%' OR p.page LIKE '%%/20__/_%%/%%') AS is_blog, " "(p.page LIKE '%%?%%' OR p.page LIKE '%%&%%') AS is_chaotic, " "c.profile, c.confidence, c.order_score, c.chaos_score " "FROM pagelist p " "LEFT JOIN url_intent_cache c " "ON c.host_id=p.host_id AND c.page_id=p.id " "AND c.updated_at >= DATE_SUB(NOW(), INTERVAL %d DAY) " "WHERE p.host_id=%d", OWS_URL_INTENT_CACHE_TTL_DAYS, host.host_id), sizeof(sqlQuery))) { FREE(tmpRes); CalcPageRankLegacy(host, hostrank); printf("OK (legacy)\r\n"); return 1; } my_mysql_ping(&gMysqlDB2, BLOCKINDEX); if(my_mysql_query_and_store_results(&gMysqlDB2, sqlQuery, tmpRes, &gRes, BLOCKINDEX) != 0) { FREE(tmpRes); CalcPageRankLegacy(host, hostrank); printf("OK (legacy)\r\n"); return 1; } res = ows_rank_guard_tmp_result(tmpRes); if(res == NULL) { FREE(tmpRes); CalcPageRankLegacy(host, hostrank); printf("OK (legacy)\r\n"); return 1; } if(!ows_rank_guard_mysql_fields(res, 13U)) { mysql_free_result(res); FREE(tmpRes); CalcPageRankLegacy(host, hostrank); printf("OK (legacy: field guard)\r\n"); return 1; } if(!ows_rank_mysql_rows_to_long((unsigned long long)mysql_num_rows(res), &nPages)) { mysql_free_result(res); FREE(tmpRes); CalcPageRankLegacy(host, hostrank); printf("OK (legacy)\r\n"); return 1; } if(!ows_rank_guard_positive_long(nPages)) { mysql_free_result(res); FREE(tmpRes); printf("OK (no pages)\r\n"); return 1; } if(!ows_rank_guard_page_count(nPages)) { mysql_free_result(res); FREE(tmpRes); CalcPageRankLegacy(host, hostrank); printf("OK (legacy: page guard)\r\n"); return 1; } if(!ows_rank_alloc_count_ok(nPages, sizeof(double)) || !ows_rank_alloc_count_ok(nPages, sizeof(int))) { mysql_free_result(res); FREE(tmpRes); CalcPageRankLegacy(host, hostrank); printf("OK (legacy)\r\n"); return 1; } OWS_RANK_ALLOC_ARRAY_OR_GOTO(ids, nPages, int, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(ranks, nPages, int, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(levels, nPages, int, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(titleLens, nPages, int, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(textLens, nPages, int, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(pageWLevel, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(pageWTitle, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(pageWText, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(pageStdLevel, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(pageStdTitle, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(pageStdText, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(pageOrder, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(pageChaos, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(accScores, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(accSqScores, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(minScores, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(maxScores, nPages, double, rank_alloc_fail); OWS_RANK_ALLOC_ARRAY_OR_GOTO(mcCounts, nPages, int, rank_alloc_fail); goto rank_alloc_ok; rank_alloc_fail: OWS_RANK_FREE_PAGE_ARRAYS(); mysql_free_result(res); FREE(tmpRes); CalcPageRankLegacy(host, hostrank); printf("OK (legacy)\r\n"); return 1; rank_alloc_ok: idx = 0; maxLevel = 1; maxTitleLen = 1; maxTextLen = 1; while((row = mysql_fetch_row(res)) != NULL && idx < nPages) { int pid; int lvl; int tlen; int xlen; int isPdf; int isEcom; int isDocs; int isBlog; int isChaotic; double pageConf = 0.0; double order = 0.0; double chaos = 0.0; double guessConf = 0.0; double guessOrder = 0.0; double guessChaos = 0.0; char guessProfileName[OWS_RANK_PROFILE_NAME_MAX]; const char *cacheProfile = ows_rank_row_text_or_null(row, 9U); const char *cacheConf = ows_rank_row_text_or_null(row, 10U); const char *cacheOrder = ows_rank_row_text_or_null(row, 11U); const char *cacheChaos = ows_rank_row_text_or_null(row, 12U); struct ows_rank_profile_cfg *pageProfile = NULL; struct ows_rank_weights_cfg blendedW; ows_rank_row_parse_int(row, 0U, 0, &pid); ows_rank_row_parse_int(row, 1U, 1, &lvl); ows_rank_row_parse_int(row, 2U, 0, &tlen); ows_rank_row_parse_int(row, 3U, 0, &xlen); ows_rank_row_parse_int(row, 4U, 0, &isPdf); ows_rank_row_parse_int(row, 5U, 0, &isEcom); ows_rank_row_parse_int(row, 6U, 0, &isDocs); ows_rank_row_parse_int(row, 7U, 0, &isBlog); ows_rank_row_parse_int(row, 8U, 0, &isChaotic); if(!ows_rank_guard_positive_int(pid)) continue; if(!ows_rank_guard_level(lvl)) lvl = 1; if(!ows_rank_guard_length_int(tlen)) tlen = 0; if(!ows_rank_guard_length_int(xlen)) xlen = 0; #if OWS_RANK_DIAG_DUP_PID if(ows_rank_guard_positive_int(pid)) { unsigned int slot = ((unsigned int)pid) & 4095U; unsigned int word = slot >> 6; unsigned int bit = slot & 63U; unsigned long long mask = 1ULL << bit; if(pidSeenMod[word] & mask) dupPidMod++; else pidSeenMod[word] |= mask; } #endif ows_guess_profile_for_url_flags(isPdf, isEcom, isDocs, isBlog, isChaotic, lvl, tlen, xlen, guessProfileName, sizeof(guessProfileName), &guessConf, &guessOrder, &guessChaos); if(ows_rank_guard_profile_name(cacheProfile)) { ows_norm_profile_name(cacheProfile, pageProfileName, sizeof(pageProfileName)); ows_rank_parse_double(cacheConf, 0.5, &pageConf); ows_rank_parse_double(cacheOrder, 0.0, &order); ows_rank_parse_double(cacheChaos, 0.0, &chaos); pageConf = ows_clip(pageConf, 0.0, 1.0); order = ows_clip(order, 0.0, 1.0); chaos = ows_clip(chaos, 0.0, 1.0); if(ows_absd(chaos - guessChaos) > 0.35 || ows_absd(order - guessOrder) > 0.35) { ows_rank_copy_string(pageProfileName, sizeof(pageProfileName), guessProfileName); pageConf = guessConf; order = guessOrder; chaos = guessChaos; if(ows_url_cache_append(host.host_id, pid, pageProfileName, pageConf, order, chaos, sqlQuery, cacheValBuf, sizeof(cacheValBuf), &cacheBatchCount, &cacheBatchFlushes, &cacheRowsBatched, &cacheBatchMin, &cacheBatchMax, &cacheFlushByCount, &cacheFlushBy75, &cacheFlushByCap, &cacheFlushAnomaly, &cacheAppendSkipInvalid, &cacheAppendSkipFormat, &cacheAppendSkipNoSpace)) cacheAppendOk++; else cacheAppendSkip++; cacheMiss++; cacheMissForced++; } else cacheHit++; } else { ows_rank_copy_string(pageProfileName, sizeof(pageProfileName), guessProfileName); pageConf = guessConf; order = guessOrder; chaos = guessChaos; if(ows_url_cache_append(host.host_id, pid, pageProfileName, pageConf, order, chaos, sqlQuery, cacheValBuf, sizeof(cacheValBuf), &cacheBatchCount, &cacheBatchFlushes, &cacheRowsBatched, &cacheBatchMin, &cacheBatchMax, &cacheFlushByCount, &cacheFlushBy75, &cacheFlushByCap, &cacheFlushAnomaly, &cacheAppendSkipInvalid, &cacheAppendSkipFormat, &cacheAppendSkipNoSpace)) cacheAppendOk++; else cacheAppendSkip++; cacheMiss++; } pageProfile = ows_profiles_find(&profilesCfg, pageProfileName); if(pageProfile == NULL) pageProfile = defaultProfile; if(pageProfile == NULL) pageProfile = &defaultFallback; ows_rank_weights_blend(&blendedW, &pageProfile->w, &defaultProfile->w, pageConf); ids[idx] = pid; levels[idx] = lvl; titleLens[idx] = tlen; textLens[idx] = xlen; pageWLevel[idx] = blendedW.w_level; pageWTitle[idx] = blendedW.w_title; pageWText[idx] = blendedW.w_text; { double uncertaintyAmp = 1.0 + (0.50 * (1.0 - pageConf)) + (0.25 * chaos); double stdLevel = ows_clip(blendedW.std_level * uncertaintyAmp, 0.0, 0.5); double stdTitle = ows_clip(blendedW.std_title * uncertaintyAmp, 0.0, 0.5); double stdText = ows_clip(blendedW.std_text * uncertaintyAmp, 0.0, 0.5); ows_rank_limit_profile_amplitude(blendedW.w_level, blendedW.w_title, blendedW.w_text, &stdLevel, &stdTitle, &stdText); pageStdLevel[idx] = stdLevel; pageStdTitle[idx] = stdTitle; pageStdText[idx] = stdText; } pageOrder[idx] = order; pageChaos[idx] = chaos; ranks[idx] = 1; accScores[idx] = 0.0; accSqScores[idx] = 0.0; minScores[idx] = 2.0; maxScores[idx] = -1.0; mcCounts[idx] = 0; if(lvl > maxLevel) maxLevel = lvl; if(tlen > maxTitleLen) maxTitleLen = tlen; if(xlen > maxTextLen) maxTextLen = xlen; idx++; } if(cacheBatchCount > 0) ows_url_cache_flush(sqlQuery, cacheValBuf, &cacheBatchCount, &cacheBatchFlushes, &cacheRowsBatched, &cacheBatchMin, &cacheBatchMax, OWS_URL_CACHE_FLUSH_FINAL, &cacheFlushByCount, &cacheFlushBy75, &cacheFlushByCap, &cacheFlushAnomaly, &cacheFlushFinal, &cacheRowsFinal); mysql_free_result(res); FREE(tmpRes); nPages = idx; if(!ows_rank_guard_positive_long(nPages)) { OWS_RANK_FREE_PAGE_ARRAYS(); CalcPageRankLegacy(host, hostrank); printf("OK (legacy)\r\n"); return 1; } /* 2) Monte Carlo + linear SVM-like robust score. We perturb weights/features, trim outlier rounds, and estimate rank stability. Low-confidence page intent and chaotic URLs widen the perturbation band. */ rounds = OWS_RANK_MC_ROUNDS; if(rounds < OWS_RANK_MC_ROUNDS_MIN) rounds = OWS_RANK_MC_ROUNDS_MIN; if(rounds > OWS_RANK_MC_ROUNDS_MAX) rounds = OWS_RANK_MC_ROUNDS_MAX; if(!ows_rank_guard_rounds(rounds)) rounds = OWS_RANK_MC_ROUNDS_MIN; pickedProfile = ows_pick_profile_for_host(host.host_id); baseWLevel = pickedProfile.w.w_level; baseWTitle = pickedProfile.w.w_title; baseWText = pickedProfile.w.w_text; ampWLevel = pickedProfile.w.std_level; ampWTitle = pickedProfile.w.std_title; ampWText = pickedProfile.w.std_text; rngState = (unsigned int)(host.host_id * 2654435761U) ^ 0x9e3779b9U; if(rngState == 0U) rngState = 0xA341316CU; rngW = rngState ^ 0xBADC0FFEU; rngF = rngState ^ 0xC001D00DU; if(rngW == 0U) rngW = 0x9E3779B1U; if(rngF == 0U) rngF = 0x85EBCA77U; maxTitleLog = ows_log1p_fast_int(maxTitleLen); maxTextLog = ows_log1p_fast_int(maxTextLen); if(!ows_rank_guard_positive_double(maxTitleLog)) maxTitleLog = 1.0; if(!ows_rank_guard_positive_double(maxTextLog)) maxTextLog = 1.0; for(i = 0; i < rounds; i++) { long j; unsigned int rngFRound = rngF ^ ((unsigned int)(i + 1) * 0x9E3779B9U); unsigned int sampleNo = (unsigned int)i + 1U; double scenarioLevel = ows_lowdisc_sym(sampleNo, 2U, 0.020); double scenarioContent = ows_lowdisc_sym(sampleNo, 3U, 0.025); double scenarioOrder = ows_lowdisc_sym(sampleNo, 5U, 0.020); double scenarioChaos = ows_lowdisc_sym(sampleNo, 7U, 0.020); for(j = 0; j < nPages; j++) { unsigned int rngFPage = rngFRound ^ ((unsigned int)ids[j] * 0x85EBCA6BU); double fLevel = (double)(maxLevel - levels[j] + 1) / (double)maxLevel; double fTitle = ows_log1p_fast_int(titleLens[j]) / maxTitleLog; double fText = ows_log1p_fast_int(textLens[j]) / maxTextLog; double orderSample; double chaosSample; double localWeightUncertainty; double localFeatureAmp; double localOrderAmp; double wLevel = pageWLevel[j] + ows_rand_sym_local(&rngW, pageStdLevel[j]); double wTitle = pageWTitle[j] + ows_rand_sym_local(&rngW, pageStdTitle[j]); double wText = pageWText[j] + ows_rand_sym_local(&rngW, pageStdText[j]); double wSum; double margin; if(wLevel < 0.01) wLevel = 0.01; if(wTitle < 0.01) wTitle = 0.01; if(wText < 0.01) wText = 0.01; wSum = wLevel + wTitle + wText; if(wSum <= 0.0 || wSum != wSum) { wLevel = 0.3333333333333333; wTitle = 0.3333333333333333; wText = 0.3333333333333333; } else { wLevel /= wSum; wTitle /= wSum; wText /= wSum; } localWeightUncertainty = ows_clip(pageStdLevel[j] + pageStdTitle[j] + pageStdText[j], 0.0, 0.35); localFeatureAmp = ows_clip(0.015 + (0.030 * pageChaos[j]) + (0.050 * localWeightUncertainty), 0.010, 0.060); localOrderAmp = ows_clip(0.010 + (0.020 * pageChaos[j]), 0.005, 0.035); fLevel = ows_clamp01(fLevel + scenarioLevel + ows_rand_sym_local(&rngFPage, localFeatureAmp)); fTitle = ows_clamp01(fTitle + scenarioContent + ows_rand_sym_local(&rngFPage, localFeatureAmp)); fText = ows_clamp01(fText + scenarioContent + ows_rand_sym_local(&rngFPage, localFeatureAmp)); orderSample = ows_clamp01(pageOrder[j] + scenarioOrder + ows_rand_sym_local(&rngFPage, localOrderAmp)); chaosSample = ows_clamp01(pageChaos[j] + scenarioChaos + ows_rand_sym_local(&rngFPage, localOrderAmp)); if(!ows_rank_mc_value_ok(fLevel) || !ows_rank_mc_value_ok(fTitle) || !ows_rank_mc_value_ok(fText) || !ows_rank_mc_value_ok(orderSample) || !ows_rank_mc_value_ok(chaosSample) || !ows_rank_mc_value_ok(wLevel) || !ows_rank_mc_value_ok(wTitle) || !ows_rank_mc_value_ok(wText)) { mcRejectedSamples++; continue; } margin = (wLevel * fLevel) + (wTitle * fTitle) + (wText * fText); margin += (0.05 * orderSample) - (0.05 * chaosSample); if(margin < 0.0) margin = 0.0; if(margin > 1.0) margin = 1.0; if(!ows_rank_mc_value_ok(margin)) { mcRejectedSamples++; continue; } accScores[j] += margin; accSqScores[j] += (margin * margin); if(margin < minScores[j]) minScores[j] = margin; if(margin > maxScores[j]) maxScores[j] = margin; mcCounts[j]++; mcAcceptedSamples++; } } /* 3) Build robust rank and Monte Carlo stability. */ meanScore = 0.0; avgStd = 0.0; avgStdErr = 0.0; maxStd = 0.0; ows_rank_reset_buffer(caseBuf, (size_t)MAXQUERYSIZE); ows_rank_reset_buffer(idBuf, (size_t)MAXQUERYSIZE); lenCase = 0; lenId = 0; batchCount = 0; for(i = 0; i < nPages; i++) { int rnk; int depthBonus; int validRounds = mcCounts[i]; double baseLevel = (double)(maxLevel - levels[i] + 1) / (double)maxLevel; double baseTitle = ows_log1p_fast_int(titleLens[i]) / maxTitleLog; double baseText = ows_log1p_fast_int(textLens[i]) / maxTextLog; double meanFull; double robust; double mcVar; double mcStd; double rankScore; char rowCase[64]; char rowId[24]; if(validRounds < OWS_RANK_MC_ROUNDS_MIN) { double fallback = ows_rank_deterministic_score(baseLevel, baseTitle, baseText, pageWLevel[i], pageWTitle[i], pageWText[i], pageOrder[i], pageChaos[i]); accScores[i] = fallback; accSqScores[i] = fallback * fallback; minScores[i] = fallback; maxScores[i] = fallback; validRounds = 1; mcFallbackPages++; } meanFull = accScores[i] / (double)validRounds; robust = ows_rank_mc_trimmed_mean(accScores[i], minScores[i], maxScores[i], validRounds); mcVar = (accSqScores[i] / (double)validRounds) - (meanFull * meanFull); if(mcVar < 0.0) mcVar = 0.0; if(validRounds > 1) mcVar = mcVar * ((double)validRounds / (double)(validRounds - 1)); mcStd = ows_sqrt_fast(mcVar); rankScore = ows_rank_mc_risk_adjusted_score(robust, mcStd); meanScore += rankScore; avgStd += mcStd; avgStdErr += mcStd / ows_sqrt_fast((double)validRounds); avgAcceptedRounds += (double)validRounds; if(mcStd > maxStd) maxStd = mcStd; depthBonus = (int)(((double)MAXPRLEV / 6.0) * ((double)(maxLevel - levels[i] + 1) / (double)maxLevel)); if(depthBonus < 0) depthBonus = 0; if(depthBonus > (MAXPRLEV/6)) depthBonus = (MAXPRLEV/6); rnk = ows_rank_safe_page_rank(hostrank, rankScore, depthBonus); ranks[i] = rnk; accScores[i] = rankScore; if(!ows_rank_guard_sql_piece(snprintf(rowCase, sizeof(rowCase), "WHEN %d THEN %d ", ids[i], ranks[i]), sizeof(rowCase))) continue; if(!ows_rank_guard_sql_piece(snprintf(rowId, sizeof(rowId), "%d", ids[i]), sizeof(rowId))) continue; if(!ows_try_append_rank_row(host.host_id, sqlQuery, caseBuf, &lenCase, idBuf, &lenId, &batchCount, rowCase, rowId)) continue; } if(batchCount > 0) { ows_flush_rank_case_batch(host.host_id, sqlQuery, caseBuf, idBuf); ows_rank_reset_buffer(caseBuf, (size_t)MAXQUERYSIZE); ows_rank_reset_buffer(idBuf, (size_t)MAXQUERYSIZE); lenCase = 0; lenId = 0; batchCount = 0; } meanScore = meanScore / (double)nPages; varScore = 0.0; for(i = 0; i < nPages; i++) { double d = accScores[i] - meanScore; varScore += d * d; } varScore = varScore / (double)nPages; avgStd = avgStd / (double)nPages; avgStdErr = avgStdErr / (double)nPages; avgAcceptedRounds = avgAcceptedRounds / (double)nPages; if((mcAcceptedSamples + mcRejectedSamples) > 0) mcAcceptanceRate = 100.0 * (double)mcAcceptedSamples / (double)(mcAcceptedSamples + mcRejectedSamples); stabilityScale = 0.5 * ows_sqrt_fast(((double)(rounds - 1) / (double)rounds)); if(stabilityScale <= 0.0) stabilityScale = 0.5; stabilityRatio = avgStd / stabilityScale; if(stabilityRatio < 0.0) stabilityRatio = 0.0; if(stabilityRatio > 1.0) stabilityRatio = 1.0; stability = 100.0 * (1.0 - stabilityRatio); printf("OK\r\n"); printf(" - MC/SVM rank stability: %.1f%% (pages: %ld, rounds: %d, avg_std: %.4f, avg_stderr: %.4f, max_std: %.4f, scale: %.4f)\r\n", stability, nPages, rounds, avgStd, avgStdErr, maxStd, stabilityScale); printf(" - MC/SVM samples: accepted=%ld rejected=%ld acceptance=%.1f%% fallback_pages=%ld avg_valid_rounds=%.1f\r\n", mcAcceptedSamples, mcRejectedSamples, mcAcceptanceRate, mcFallbackPages, avgAcceptedRounds); printf(" - Learned weights: level=%.4f title=%.4f text=%.4f (std: %.4f/%.4f/%.4f, source: %s)\r\n", baseWLevel, baseWTitle, baseWText, ampWLevel, ampWTitle, ampWText, pickedProfile.w.loaded ? "rank_weights.conf" : "defaults"); printf(" - LTR profile: %s (selector: %s, confidence: %.2f, conf_file: %s)\r\n", pickedProfile.name, pickedProfile.source, pickedProfile.confidence, pickedProfile.conf_file_loaded ? "loaded" : "missing"); printf(" - url_intent_cache: hit=%ld miss=%ld forced=%ld rate=%.1f%% ttl=%d days\r\n", cacheHit, cacheMiss, cacheMissForced, (cacheHit + cacheMiss) > 0 ? (100.0 * (double)cacheHit / (double)(cacheHit + cacheMiss)) : 0.0, OWS_URL_INTENT_CACHE_TTL_DAYS); printf(" - url_intent_cache writes: rows_batched=%ld batches=%ld batch_rows[min/avg/max]=%d/%.1f/%d\r\n", cacheRowsBatched, cacheBatchFlushes, cacheBatchMin, (cacheBatchFlushes > 0) ? ((double)cacheRowsBatched / (double)cacheBatchFlushes) : 0.0, cacheBatchMax); cacheAppendAttempted = cacheAppendOk + cacheAppendSkip; skipCauses = cacheAppendSkipInvalid + cacheAppendSkipFormat + cacheAppendSkipNoSpace; attemptedPerMiss = (cacheMiss > 0) ? ((double)cacheAppendAttempted / (double)cacheMiss) : 0.0; printf(" - url_intent_cache writes_rate: attempted=%ld effective=%ld effective_rate=%.1f%% pages=%ld skip_rate=%.1f%%\r\n", cacheAppendAttempted, cacheAppendOk, (nPages > 0) ? (100.0 * (double)cacheAppendOk / (double)nPages) : 0.0, nPages, (cacheAppendAttempted > 0) ? (100.0 * (double)cacheAppendSkip / (double)cacheAppendAttempted) : 0.0); printf(" - url_intent_cache attempted_per_miss: %.3f (attempted=%ld miss=%ld forced=%ld)\r\n", attemptedPerMiss, cacheAppendAttempted, cacheMiss, cacheMissForced); #if OWS_RANK_DIAG_DUP_PID printf(" - url_intent_cache pid_dups_mod4096: %ld (debug)\r\n", dupPidMod); #endif printf(" - url_intent_cache append: ok=%ld skip=%ld (invalid=%ld format=%ld nospace=%ld)\r\n", cacheAppendOk, cacheAppendSkip, cacheAppendSkipInvalid, cacheAppendSkipFormat, cacheAppendSkipNoSpace); printf(" - url_intent_cache flush_reasons: batch=%ld buffer75=%ld cap=%ld anomaly=%ld\r\n", cacheFlushByCount, cacheFlushBy75, cacheFlushByCap, cacheFlushAnomaly); printf(" - url_intent_cache flush_final: count=%ld rows=%ld\r\n", cacheFlushFinal, cacheRowsFinal); if(cacheAppendOk > cacheMiss) fprintf(stderr, "[OWS][rank] WARNING: cacheAppendOk(%ld) > cacheMiss(%ld)\n", cacheAppendOk, cacheMiss); if(cacheRowsFinal > cacheRowsBatched) fprintf(stderr, "[OWS][rank] WARNING: cacheRowsFinal(%ld) > cacheRowsBatched(%ld)\n", cacheRowsFinal, cacheRowsBatched); if(cacheFlushFinal > 1) fprintf(stderr, "[OWS][rank] WARNING: cacheFlushFinal=%ld (expected 0/1 per host run)\n", cacheFlushFinal); if(cacheAppendSkip != skipCauses) fprintf(stderr, "[OWS][rank] WARNING: cacheAppendSkip(%ld) != skip_causes_sum(%ld)\n", cacheAppendSkip, skipCauses); if(cacheMiss > 0 && attemptedPerMiss > 1.05) fprintf(stderr, "[OWS][rank] WARNING: attempted_per_miss=%.3f (>1.05), possible double-attempt or misaligned miss counter\n", attemptedPerMiss); #if OWS_RANK_DIAG_DUP_PID if(cacheMiss > 0 && attemptedPerMiss > 1.05 && dupPidMod > 0) fprintf(stderr, "[OWS][rank] WARNING: pid_dups_mod4096=%ld, potential duplicate page rows in source query\n", dupPidMod); #endif if(cacheMiss > 0 && attemptedPerMiss < 0.95) fprintf(stderr, "[OWS][rank] WARNING: attempted_per_miss=%.3f (<0.95), miss counted without append attempt\n", attemptedPerMiss); ows_save_rank_stability_host(host.host_id, stability, avgStd, maxStd, rounds, meanScore, varScore, pickedProfile.name, pickedProfile.source, pickedProfile.confidence); OWS_RANK_FREE_PAGE_ARRAYS(); return 1; } #endif /*EOF*/