193 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * GRU KERNEL MCS INSTRUCTIONS4 *5 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.6 */7 8#include <linux/kernel.h>9#include "gru.h"10#include "grulib.h"11#include "grutables.h"12 13/* 10 sec */14#include <linux/sync_core.h>15#include <asm/tsc.h>16#define GRU_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000)17#define CLKS2NSEC(c) ((c) * 1000000 / tsc_khz)18 19/* Extract the status field from a kernel handle */20#define GET_MSEG_HANDLE_STATUS(h) (((*(unsigned long *)(h)) >> 16) & 3)21 22struct mcs_op_statistic mcs_op_statistics[mcsop_last];23 24static void update_mcs_stats(enum mcs_op op, unsigned long clks)25{26 unsigned long nsec;27 28 nsec = CLKS2NSEC(clks);29 atomic_long_inc(&mcs_op_statistics[op].count);30 atomic_long_add(nsec, &mcs_op_statistics[op].total);31 if (mcs_op_statistics[op].max < nsec)32 mcs_op_statistics[op].max = nsec;33}34 35static void start_instruction(void *h)36{37 unsigned long *w0 = h;38 39 wmb(); /* setting CMD/STATUS bits must be last */40 *w0 = *w0 | 0x20001;41 gru_flush_cache(h);42}43 44static void report_instruction_timeout(void *h)45{46 unsigned long goff = GSEGPOFF((unsigned long)h);47 char *id = "???";48 49 if (TYPE_IS(CCH, goff))50 id = "CCH";51 else if (TYPE_IS(TGH, goff))52 id = "TGH";53 else if (TYPE_IS(TFH, goff))54 id = "TFH";55 56 panic(KERN_ALERT "GRU %p (%s) is malfunctioning\n", h, id);57}58 59static int wait_instruction_complete(void *h, enum mcs_op opc)60{61 int status;62 unsigned long start_time = get_cycles();63 64 while (1) {65 cpu_relax();66 status = GET_MSEG_HANDLE_STATUS(h);67 if (status != CCHSTATUS_ACTIVE)68 break;69 if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time)) {70 report_instruction_timeout(h);71 start_time = get_cycles();72 }73 }74 if (gru_options & OPT_STATS)75 update_mcs_stats(opc, get_cycles() - start_time);76 return status;77}78 79int cch_allocate(struct gru_context_configuration_handle *cch)80{81 int ret;82 83 cch->opc = CCHOP_ALLOCATE;84 start_instruction(cch);85 ret = wait_instruction_complete(cch, cchop_allocate);86 87 /*88 * Stop speculation into the GSEG being mapped by the previous ALLOCATE.89 * The GSEG memory does not exist until the ALLOCATE completes.90 */91 sync_core();92 return ret;93}94 95int cch_start(struct gru_context_configuration_handle *cch)96{97 cch->opc = CCHOP_START;98 start_instruction(cch);99 return wait_instruction_complete(cch, cchop_start);100}101 102int cch_interrupt(struct gru_context_configuration_handle *cch)103{104 cch->opc = CCHOP_INTERRUPT;105 start_instruction(cch);106 return wait_instruction_complete(cch, cchop_interrupt);107}108 109int cch_deallocate(struct gru_context_configuration_handle *cch)110{111 int ret;112 113 cch->opc = CCHOP_DEALLOCATE;114 start_instruction(cch);115 ret = wait_instruction_complete(cch, cchop_deallocate);116 117 /*118 * Stop speculation into the GSEG being unmapped by the previous119 * DEALLOCATE.120 */121 sync_core();122 return ret;123}124 125int cch_interrupt_sync(struct gru_context_configuration_handle126 *cch)127{128 cch->opc = CCHOP_INTERRUPT_SYNC;129 start_instruction(cch);130 return wait_instruction_complete(cch, cchop_interrupt_sync);131}132 133int tgh_invalidate(struct gru_tlb_global_handle *tgh,134 unsigned long vaddr, unsigned long vaddrmask,135 int asid, int pagesize, int global, int n,136 unsigned short ctxbitmap)137{138 tgh->vaddr = vaddr;139 tgh->asid = asid;140 tgh->pagesize = pagesize;141 tgh->n = n;142 tgh->global = global;143 tgh->vaddrmask = vaddrmask;144 tgh->ctxbitmap = ctxbitmap;145 tgh->opc = TGHOP_TLBINV;146 start_instruction(tgh);147 return wait_instruction_complete(tgh, tghop_invalidate);148}149 150int tfh_write_only(struct gru_tlb_fault_handle *tfh,151 unsigned long paddr, int gaa,152 unsigned long vaddr, int asid, int dirty,153 int pagesize)154{155 tfh->fillasid = asid;156 tfh->fillvaddr = vaddr;157 tfh->pfn = paddr >> GRU_PADDR_SHIFT;158 tfh->gaa = gaa;159 tfh->dirty = dirty;160 tfh->pagesize = pagesize;161 tfh->opc = TFHOP_WRITE_ONLY;162 start_instruction(tfh);163 return wait_instruction_complete(tfh, tfhop_write_only);164}165 166void tfh_write_restart(struct gru_tlb_fault_handle *tfh,167 unsigned long paddr, int gaa,168 unsigned long vaddr, int asid, int dirty,169 int pagesize)170{171 tfh->fillasid = asid;172 tfh->fillvaddr = vaddr;173 tfh->pfn = paddr >> GRU_PADDR_SHIFT;174 tfh->gaa = gaa;175 tfh->dirty = dirty;176 tfh->pagesize = pagesize;177 tfh->opc = TFHOP_WRITE_RESTART;178 start_instruction(tfh);179}180 181void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh)182{183 tfh->opc = TFHOP_USER_POLLING_MODE;184 start_instruction(tfh);185}186 187void tfh_exception(struct gru_tlb_fault_handle *tfh)188{189 tfh->opc = TFHOP_EXCEPTION;190 start_instruction(tfh);191}192 193