多核环境下cache line的测试

CPU1读取了数据a(假设a小于cache行大小),并存入CPU1的高速缓存.

CPU2也读取了数据a,并存入CPU2的高速缓存.

CPU1修改了数据a, a被放回CPU1的高速缓存行. 但是该信息并没有被写入RAM.

CPU2访问a, 但由于CPU1并未将数据写入RAM, 导致了数据不同步.

为了解决这个问题, 芯片设计者制定了一个规则. 当一个CPU修改高速缓存行中的字节时, 计算机中的其它CPU会被通知, 它们的高速缓存将视为无效. 于是, 在上面的情况下, CPU2发现自己的高速缓存中数据已无效, CPU1将立即把自己的数据写回RAM, 然后CPU2重新读取该数据. 这样就完成了一次两个cpu之间cache的同步.

为了测试上述场景, 我编写了如下程序进行测试:

#define EXEC_COUNT (100 * 1000 * 1000)

struct bits_t
{
    int a;
    char placeholder[64];
    int b;
};

struct bits_t bits;

int which_cpu(const char* prefix_)
{
    #ifdef ENABLE_WHCIH_CPU
    cpu_set_t cur_cpu;
    CPU_ZERO(&cur_cpu);
    if (sched_getaffinity(0, sizeof(cur_cpu), &cur_cpu) == -1)
    {
        printf("warning: cound not get cpu affinity, continuing...n");
        return -1;
    }
    int num = sysconf(_SC_NPROCESSORS_CONF);
    for (int i = 0; i < num; i++)
    {
        if (CPU_ISSET(i, &cur_cpu))
        {
            printf("[%s] this process %d is running processor : %dn", prefix_, getpid(), i);
        }
    }
    #endif

    return 0;
}

int set_cpu(int cpu_id_)
{
    #ifdef ENABLE_SET_CPU
    cpu_set_t mask;
    CPU_ZERO(&mask);
    CPU_SET(cpu_id_, &mask);
    if (sched_setaffinity(0, sizeof(mask), &mask) == -1)
    {
        printf("warning: could not set CPU affinity, continuing...n");
        return -1;
    }
    #endif

    return 0;
}

void* thd_func1(void* arg_)
{
    set_cpu(0);
    which_cpu("thread 1 start");
    timeval begin_tv;
    gettimeofday(&begin_tv, NULL);

    for (int i = 0; i < EXEC_COUNT; i++)
    {
        bits.a += 1;
        int a = bits.a;
    }

    timeval end_tv;
    gettimeofday(&end_tv, NULL);
    printf("thd1 perf:[%lu]usn", (end_tv.tv_sec * 1000 * 1000 + end_tv.tv_usec) - (begin_tv.tv_sec * 1000 * 1000 + begin_tv.tv_usec));
    which_cpu("thread 1 end");

    return NULL;
}

void* thd_func2(void* arg_)
{
    set_cpu(1);
    which_cpu("thread 2 start");
    timeval begin_tv;
    gettimeofday(&begin_tv, NULL);

    for (int i = 0; i < EXEC_COUNT; i++)
    {
        bits.b += 2;
        int b = bits.b;
    }

    timeval end_tv;
    gettimeofday(&end_tv, NULL);
    printf("thd2 perf:[%lu]usn", (end_tv.tv_sec * 1000 * 1000 + end_tv.tv_usec) - (begin_tv.tv_sec * 1000 * 1000 + begin_tv.tv_usec));
    which_cpu("thread 2 end");

    return NULL;
}

int main(int argc_, char* argv_[])
{
    int num = sysconf(_SC_NPROCESSORS_CONF);
    printf("system has %d processor(s).n", num);
    cpu_set_t cpu_mask;
    cpu_set_t cur_cpu_info;

    memset((void*)&bits, 0, sizeof(bits_t));
    set_cpu(0);
    which_cpu("main thread");

    pthread_t pid1;
    pthread_create(&pid1, NULL, thd_func1, NULL);

    pthread_t pid2;
    pthread_create(&pid2, NULL, thd_func2, NULL);

    pthread_join(pid1, NULL);
    pthread_join(pid2, NULL);

    return 0;
}

情况二测试结果:

情况四测试结果: