/* Reading UTF-8 by hand — the rule, and what must be rejected (RFC 3629). */
#include <stdio.h>
#include <stddef.h>
#include <string.h>

/* The whole rule of UTF-8 fits in one table.
     0xxxxxxx                            -> 1 byte,  U+0000..U+007F
     110xxxxx 10xxxxxx                   -> 2 bytes, U+0080..U+07FF
     1110xxxx 10xxxxxx 10xxxxxx          -> 3 bytes, U+0800..U+FFFF
     11110xxx 10xxxxxx 10xxxxxx 10xxxxxx -> 4 bytes, U+10000..U+10FFFF
   A trail byte is always 10xxxxxx. So from any byte you can tell whether you
   are at the start of a character or inside one — that is self-synchronisation. */

typedef enum { OK, BAD_LEAD, BAD_TRAIL, TRUNCATED, OVERLONG, SURROGATE, TOO_BIG } verdict;

static const char *why(verdict v)
{
    switch (v) {
    case OK:        return "fine";
    case BAD_LEAD:  return "not a value that can lead a character";
    case BAD_TRAIL: return "a trail byte is not 10xxxxxx";
    case TRUNCATED: return "not enough bytes";
    case OVERLONG:  return "overlong — this could be written shorter";
    case SURROGATE: return "surrogates (U+D800..DFFF) cannot be encoded";
    case TOO_BIG:   return "beyond U+10FFFF";
    }
    return "?";
}

/* Read one character. On success the byte count goes into *len. */
static verdict decode(const unsigned char *s, size_t n,
                      unsigned long *cp, size_t *len)
{
    if (n == 0) return TRUNCATED;
    unsigned char c = s[0];
    size_t need;
    unsigned long v;

    if (c < 0x80)                  { need = 1; v = c; }
    else if ((c & 0xE0) == 0xC0)   { need = 2; v = c & 0x1Fu; }
    else if ((c & 0xF0) == 0xE0)   { need = 3; v = c & 0x0Fu; }
    else if ((c & 0xF8) == 0xF0)   { need = 4; v = c & 0x07u; }
    else return BAD_LEAD;          /* starts 10xxxxxx, or 11111xxx */

    if (n < need) return TRUNCATED;
    for (size_t i = 1; i < need; i++) {
        if ((s[i] & 0xC0) != 0x80) return BAD_TRAIL;
        v = (v << 6) | (unsigned long)(s[i] & 0x3Fu);
    }

    /* From here on: shapes that parse but must not be accepted */
    static const unsigned long lowest[5] = { 0, 0, 0x80, 0x800, 0x10000 };
    if (v < lowest[need])                return OVERLONG;
    if (v >= 0xD800UL && v <= 0xDFFFUL)  return SURROGATE;
    if (v > 0x10FFFFUL)                  return TOO_BIG;

    *cp = v; *len = need;
    return OK;
}

static void scan(const char *label, const unsigned char *s, size_t n)
{
    printf("\n[%s]", label);
    for (size_t i = 0; i < n; i++) printf(" %02X", s[i]);
    puts("");

    for (size_t pos = 0; pos < n; ) {
        unsigned long cp = 0;
        size_t len = 0;
        verdict v = decode(s + pos, n - pos, &cp, &len);
        if (v == OK) {
            printf("  %zu byte%s -> U+%04lX\n", len, len == 1 ? "" : "s", cp);
            pos += len;
        } else {
            printf("  rejected: %s\n", why(v));
            pos += 1;                /* drop one byte and resynchronise */
            break;
        }
    }
}

int main(void)
{
    /* the good ones */
    scan("ASCII", (const unsigned char *)"Hi", 2);
    scan("Hangul U+D55C", (const unsigned char *)"\xED\x95\x9C", 3);
    scan("emoji", (const unsigned char *)"\xF0\x9F\x98\x80", 4);

    /* the ones that must be rejected */
    scan("starts on a trail byte", (const unsigned char *)"\x9C", 1);
    scan("a truncated three-byte", (const unsigned char *)"\xED\x95", 2);
    scan("a bad trail byte", (const unsigned char *)"\xED\x41\x9C", 3);
    scan("overlong C0 80", (const unsigned char *)"\xC0\x80", 2);
    scan("surrogate ED A0 80", (const unsigned char *)"\xED\xA0\x80", 3);
    scan("out of range F5 80 80 80", (const unsigned char *)"\xF5\x80\x80\x80", 4);

    puts("\nSelf-synchronisation — point at any byte and you know where you are:");
    const unsigned char *s = (const unsigned char *)"a한글b";
    size_t n = strlen((const char *)s);
    for (size_t i = 0; i < n; i++)
        printf("  byte %zu (%02X): %s\n", i, s[i],
               (s[i] & 0xC0) == 0x80 ? "inside a character" : "start of a character");
    return 0;
}

/* "How many characters" has three answers — bytes, code points, and what a reader sees. */
#include <locale.h>
#include <stdio.h>
#include <string.h>
#include <wchar.h>

/* The full rule for "one character as a reader sees it" (a grapheme cluster)
   is Unicode UAX #29, and a proper implementation needs tables. Here we count
   with a *simplified rule* covering the four cases you meet most — the point is
   to see the concept, not to be complete.
     (1) combining marks (U+0300..U+036F and friends) join the previous one
     (2) Hangul conjoining jamo (U+1100..U+11FF) join the previous one
     (3) whatever follows a ZWJ (U+200D) joins the previous one
     (4) regional indicators (U+1F1E6..U+1F1FF) pair up into a flag */
static int is_combining(unsigned long c)
{
    return (c >= 0x0300 && c <= 0x036F)     /* combining diacritics */
        || (c >= 0x1160 && c <= 0x11FF)     /* Hangul vowels and final consonants */
        || (c >= 0xFE00 && c <= 0xFE0F)     /* variation selectors */
        || (c >= 0x1F3FB && c <= 0x1F3FF);  /* skin-tone modifiers */
}
static int is_regional(unsigned long c) { return c >= 0x1F1E6 && c <= 0x1F1FF; }

static void measure(const char *label, const char *s)
{
    size_t bytes = strlen(s);
    size_t cps = 0, clusters = 0;

    mbstate_t st;
    memset(&st, 0, sizeof st);

    unsigned long prev = 0;
    int prev_was_zwj = 0, prev_regional = 0;

    for (size_t pos = 0; pos < bytes; ) {
        wchar_t wc = 0;
        size_t r = mbrtowc(&wc, s + pos, bytes - pos, &st);
        if (r == (size_t)-1 || r == (size_t)-2) break;
        if (r == 0) r = 1;
        pos += r;
        cps++;

        unsigned long c = (unsigned long)wc;
        int joins = (cps > 1) && (is_combining(c) || prev_was_zwj || c == 0x200D
                                  || (is_regional(c) && prev_regional));
        if (!joins) clusters++;

        prev_was_zwj = (c == 0x200D);
        prev_regional = is_regional(c) && !prev_regional;
        prev = c;
    }
    (void)prev;

    printf("  bytes %2zu  code points %2zu  visible characters %2zu   %s\n",
           bytes, cps, clusters, label);
}

int main(void)
{
    const char *loc = setlocale(LC_CTYPE, "C.UTF-8");
    if (!loc) loc = setlocale(LC_CTYPE, "en_US.UTF-8");
    if (!loc) { puts("no UTF-8 locale — skipping this demonstration"); return 0; }

    puts("[the same letter, written two ways]");
    measure("\"가\" precomposed, U+AC00",       "\uAC00");
    measure("\"가\" composed, U+1100 U+1161",  "\u1100\u1161");
    measure("\"e\\u0301\", combining accent",  "e\u0301");
    measure("\"\\u00E9\", precomposed",         "\u00E9");

    puts("\n[emoji: several code points that look like one character]");
    measure("grinning face",          "\U0001F600");
    measure("family, joined by ZWJ",  "\U0001F468\u200D\U0001F469\u200D\U0001F467");
    measure("a flag, two indicators", "\U0001F1F0\U0001F1F7");
    measure("wave + skin tone",       "\U0001F44B\U0001F3FD");

    puts("\n[one sentence]");
    measure("\"Hello, 세계!\"", "Hello, 세계!");

    puts("\nSo: before asking \"how many characters\", settle *which layer*.");
    puts("  bytes = what you store and send,  code points = Unicode's unit,");
    puts("  grapheme clusters = what the cursor steps over (UAX #29).");
    puts("The count above uses four simplified rules — a complete decision");
    puts("belongs to a dedicated library such as ICU.");
    return 0;
}

/* Legacy two-byte encodings — what happens when trail bytes overlap ASCII. */
#include <stdio.h>
#include <string.h>

/* Each encoding has its own lead-byte and trail-byte ranges.
   The dangerous ones are those whose trail bytes reach into ASCII. */
static void show_table(void)
{
    puts("byte ranges per encoding (two-byte characters)");
    puts("  encoding    lead bytes         trail bytes           0x5C('\\') as trail?");
    puts("  EUC-KR      A1-FE              A1-FE                 no");
    puts("  CP949(UHC)  81-FE              41-5A,61-7A,81-FE     no");
    puts("  Shift_JIS   81-9F,E0-FC        40-7E,80-FC           * yes");
    puts("  Big5        81-FE              40-7E,A1-FE           * yes");
    puts("  GBK         81-FE              40-FE(not 7F)         * yes");
}

/* A path written in Shift_JIS: C:\表\ソ.txt
   表 = 95 5C, ソ = 83 5C — both have 0x5C as their trail byte (checked with iconv). */
static const unsigned char sjis_path[] = {
    'C', ':', 0x5C, 0x95, 0x5C, 0x5C, 0x83, 0x5C, '.', 't', 'x', 't', 0
};

/* The same in EUC-KR: C:\한글.txt  (한 = C7 D1, 글 = B1 DB) */
static const unsigned char euckr_path[] = {
    'C', ':', 0x5C, 0xC7, 0xD1, 0xB1, 0xDB, '.', 't', 'x', 't', 0
};

static int sjis_is_lead(unsigned char c)
{ return (c >= 0x81 && c <= 0x9F) || (c >= 0xE0 && c <= 0xFC); }

static int euckr_is_lead(unsigned char c) { return c >= 0xA1 && c <= 0xFE; }

/* Find the last separator, stepping over lead bytes — the correct way */
static long safe_last_sep(const unsigned char *s, int (*is_lead)(unsigned char))
{
    long last = -1;
    for (size_t i = 0; s[i]; ) {
        if (is_lead(s[i]) && s[i + 1]) { i += 2; continue; }  /* a two-byte character */
        if (s[i] == 0x5C) last = (long)i;
        i += 1;
    }
    return last;
}

static void dump(const char *label, const unsigned char *s)
{
    printf("  %-10s", label);
    for (size_t i = 0; s[i]; i++) printf(" %02X", s[i]);
    puts("");
}

static void test(const char *name, const unsigned char *path,
                 int (*is_lead)(unsigned char))
{
    printf("\n[%s]\n", name);
    dump("bytes", path);

    /* The usual way — look at bytes only, find the last '\\' */
    const char *hit = strrchr((const char *)path, 0x5C);
    long naive = hit ? (long)(hit - (const char *)path) : -1;
    long safe  = safe_last_sep(path, is_lead);

    printf("  last '\\' found by strrchr:   at %ld\n", naive);
    printf("  a lead-byte-aware scan:      at %ld\n", safe);
    if (naive == safe) puts("  -> the same. This encoding is safe here.");
    else {
        puts("  -> different! strrchr took a *trail byte* for a separator.");
        printf("     Cut there and the file name becomes \"");
        for (size_t i = (size_t)naive + 1; path[i]; i++) printf("%02X ", path[i]);
        puts("\" — a broken fragment.");
    }
}

int main(void)
{
    show_table();
    test("Shift_JIS: C:\\表\\ソ.txt", sjis_path, sjis_is_lead);
    test("EUC-KR: C:\\한글.txt", euckr_path, euckr_is_lead);

    puts("\nThis accident even has a name in Japan — dame-moji (ダメ文字),");
    puts("with 表, ソ, 十 and ダ as the usual victims. UTF-8 does not have it:");
    puts("its trail bytes are always 0x80 or above, so they never touch ASCII.");

    const unsigned char utf8_path[] = "C:\\한글.txt";
    dump("UTF-8", utf8_path);
    const char *h = strrchr((const char *)utf8_path, 0x5C);
    printf("  last '\\' found by strrchr:   at %ld — always right\n",
           h ? (long)(h - (const char *)utf8_path) : -1);
    return 0;
}