From 776c856d638fec3be9ea8830c9b610d03923f9f5 Mon Sep 17 00:00:00 2001
From: Michael Rogenmoser <michael@rogenmoser.us>
Date: Thu, 19 Jan 2023 13:23:02 +0100
Subject: [PATCH] Add initial HMR software config

---
 configs/hmr_pulp.sh                   |  17 +++
 include/archi/chips/pulp/memory_map.h |  10 ++
 include/archi/chips/pulp/pulp.h       |   4 +
 include/archi/hmr/hmr_v1.h            | 173 ++++++++++++++++++++++++++
 include/hal/chips/pulp/pulp.h         |   4 +
 include/hal/hmr/hmr_v1.h              | 101 +++++++++++++++
 kernel/cluster.c                      |   4 +
 rules/pulpos/targets/pulp.mk          |   4 +
 8 files changed, 317 insertions(+)
 create mode 100644 configs/hmr_pulp.sh
 create mode 100644 include/archi/hmr/hmr_v1.h
 create mode 100644 include/hal/hmr/hmr_v1.h

diff --git a/configs/hmr_pulp.sh b/configs/hmr_pulp.sh
new file mode 100644
index 0000000..c0ca31a
--- /dev/null
+++ b/configs/hmr_pulp.sh
@@ -0,0 +1,17 @@
+#!/bin/bash -e
+
+export PULPRT_TARGET=pulp
+export PULPRUN_TARGET=pulp
+export USE_CV32E40P=1
+export ARCHI_HMR=1
+
+if [  -n "${ZSH_VERSION:-}" ]; then 
+        DIR="$(readlink -f -- "${(%):-%x}")"
+        scriptDir="$(dirname $DIR)"
+else
+
+    scriptDir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+
+fi
+
+source $scriptDir/common.sh
diff --git a/include/archi/chips/pulp/memory_map.h b/include/archi/chips/pulp/memory_map.h
index 999f5f9..750edc4 100644
--- a/include/archi/chips/pulp/memory_map.h
+++ b/include/archi/chips/pulp/memory_map.h
@@ -99,6 +99,7 @@
 #define ARCHI_ICACHE_CTRL_OFFSET         0x00001400
 #define ARCHI_MCHAN_EXT_OFFSET           0x00001800
 #define ARCHI_IDMA_EXT_OFFSET            0x00001800
+#define ARCHI_HMR_OFFSET                 0x00002000
 
 #define ARCHI_CLUSTER_PERIPHERALS_ADDR             ( ARCHI_CLUSTER_ADDR + ARCHI_CLUSTER_PERIPHERALS_OFFSET )
 #define ARCHI_CLUSTER_PERIPHERALS_GLOBAL_ADDR(cid) ( ARCHI_CLUSTER_GLOBAL_ADDR(cid) + ARCHI_CLUSTER_PERIPHERALS_OFFSET )
@@ -109,6 +110,15 @@
 #define ARCHI_HWCE_ADDR                            ( ARCHI_CLUSTER_PERIPHERALS_ADDR + ARCHI_HWCE_OFFSET )
 #define ARCHI_MCHAN_EXT_ADDR                       ( ARCHI_CLUSTER_PERIPHERALS_ADDR + ARCHI_MCHAN_EXT_OFFSET )
 #define ARCHI_IDMA_EXT_ADDR                        ( ARCHI_CLUSTER_PERIPHERALS_ADDR + ARCHI_IDMA_EXT_OFFSET )
+#define ARCHI_HMR_ADDR                             ( ARCHI_CLUSTER_PERIPHERALS_ADDR + ARCHI_HMR_OFFSET )
+
+#define ARCHI_CLUSTER_CTRL_GLOBAL_ADDR(cid)        ( ARCHI_CLUSTER_PERIPHERALS_GLOBAL_ADDR(cid) + ARCHI_CLUSTER_CTRL_OFFSET )
+#define ARCHI_ICACHE_CTRL_GLOBAL_ADDR(cid)         ( ARCHI_CLUSTER_PERIPHERALS_GLOBAL_ADDR(cid) + ARCHI_ICACHE_CTRL_OFFSET )
+#define ARCHI_EU_GLOBAL_ADDR(cid)                  ( ARCHI_CLUSTER_PERIPHERALS_GLOBAL_ADDR(cid) + ARCHI_EU_OFFSET )
+#define ARCHI_HWCE_GLOBAL_ADDR(cid)                ( ARCHI_CLUSTER_PERIPHERALS_GLOBAL_ADDR(cid) + ARCHI_HWCE_OFFSET )
+#define ARCHI_MCHAN_EXT_GLOBAL_ADDR(cid)           ( ARCHI_CLUSTER_PERIPHERALS_GLOBAL_ADDR(cid) + ARCHI_MCHAN_EXT_OFFSET )
+#define ARCHI_IDMA_EXT_GLOBAL_ADDR(cid)            ( ARCHI_CLUSTER_PERIPHERALS_GLOBAL_ADDR(cid) + ARCHI_IDMA_EXT_OFFSET )
+#define ARCHI_HMR_GLOBAL_ADDR(cid)                 ( ARCHI_CLUSTER_PERIPHERALS_GLOBAL_ADDR(cid) + ARCHI_HMR_OFFSET )
 
 
 
diff --git a/include/archi/chips/pulp/pulp.h b/include/archi/chips/pulp/pulp.h
index e98c637..53dfe1d 100644
--- a/include/archi/chips/pulp/pulp.h
+++ b/include/archi/chips/pulp/pulp.h
@@ -51,4 +51,8 @@
 #include "archi/udma/uart/udma_uart_v1.h"
 #include "archi/udma/udma_v3.h"
 
+#ifdef ARCHI_HMR
+#include "archi/hmr/hmr_v1.h"
+#endif // ARCHI_HMR
+
 #endif
diff --git a/include/archi/hmr/hmr_v1.h b/include/archi/hmr/hmr_v1.h
new file mode 100644
index 0000000..57ae6b4
--- /dev/null
+++ b/include/archi/hmr/hmr_v1.h
@@ -0,0 +1,173 @@
+/*
+ * Copyright (C) 2023 ETH Zurich and University of Bologna
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __ARCHI_HMR_HMR_V1_H__
+#define __ARCHI_HMR_HMR_V1_H__
+
+#define HMR_IN_INTERLEAVED 1
+
+#define HMR_TOP_OFFSET 0x000
+#define HMR_CORE_OFFSET   0x100
+#define HMR_DMR_OFFSET    0x200
+#define HMR_TMR_OFFSET    0x300
+
+#define HMR_CORE_INCREMENT 0x008
+#define HMR_TMR_INCREMENT  0x010
+
+// Generated register defines for HMR_registers
+
+#ifndef _HMR_REGISTERS_REG_DEFS_
+#define _HMR_REGISTERS_REG_DEFS_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#define HMR_REGISTERS_PARAM_NUM_CORES 12
+
+#define HMR_REGISTERS_PARAM_NUM_D_M_R_GROUPS 6
+
+#define HMR_REGISTERS_PARAM_NUM_T_M_R_GROUPS 4
+
+// Register width
+#define HMR_REGISTERS_PARAM_REG_WIDTH 32
+
+// Available Configurations from implemented hardware.
+#define HMR_REGISTERS_AVAIL_CONFIG_REG_OFFSET 0x0
+#define HMR_REGISTERS_AVAIL_CONFIG_INDEPENDENT_BIT 0
+#define HMR_REGISTERS_AVAIL_CONFIG_DUAL_BIT 1
+#define HMR_REGISTERS_AVAIL_CONFIG_TRIPLE_BIT 2
+
+// Enabled cores, based on the configuration. Can be used for barriers.
+#define HMR_REGISTERS_CORES_EN_REG_OFFSET 0x4
+#define HMR_REGISTERS_CORES_EN_CORES_EN_MASK 0xfff
+#define HMR_REGISTERS_CORES_EN_CORES_EN_OFFSET 0
+#define HMR_REGISTERS_CORES_EN_CORES_EN_FIELD \
+  ((bitfield_field32_t) { .mask = HMR_REGISTERS_CORES_EN_CORES_EN_MASK, .index = HMR_REGISTERS_CORES_EN_CORES_EN_OFFSET })
+
+// DMR configuration enable, on bit per DMR group.
+#define HMR_REGISTERS_DMR_ENABLE_REG_OFFSET 0x8
+#define HMR_REGISTERS_DMR_ENABLE_DMR_ENABLE_MASK 0x3f
+#define HMR_REGISTERS_DMR_ENABLE_DMR_ENABLE_OFFSET 0
+#define HMR_REGISTERS_DMR_ENABLE_DMR_ENABLE_FIELD \
+  ((bitfield_field32_t) { .mask = HMR_REGISTERS_DMR_ENABLE_DMR_ENABLE_MASK, .index = HMR_REGISTERS_DMR_ENABLE_DMR_ENABLE_OFFSET })
+
+// TMR configuration enable, one bit per TMR group.
+#define HMR_REGISTERS_TMR_ENABLE_REG_OFFSET 0xc
+#define HMR_REGISTERS_TMR_ENABLE_TMR_ENABLE_MASK 0xf
+#define HMR_REGISTERS_TMR_ENABLE_TMR_ENABLE_OFFSET 0
+#define HMR_REGISTERS_TMR_ENABLE_TMR_ENABLE_FIELD \
+  ((bitfield_field32_t) { .mask = HMR_REGISTERS_TMR_ENABLE_TMR_ENABLE_MASK, .index = HMR_REGISTERS_TMR_ENABLE_TMR_ENABLE_OFFSET })
+
+// TMR configuration bits.
+#define HMR_REGISTERS_TMR_CONFIG_REG_OFFSET 0x10
+#define HMR_REGISTERS_TMR_CONFIG_DELAY_RESYNCH_BIT 0
+#define HMR_REGISTERS_TMR_CONFIG_SETBACK_BIT 1
+#define HMR_REGISTERS_TMR_CONFIG_RELOAD_SETBACK_BIT 2
+#define HMR_REGISTERS_TMR_CONFIG_FORCE_RESYNCH_BIT 3
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+#endif  // _HMR_REGISTERS_REG_DEFS_
+// End generated register defines for HMR_registers
+
+// Generated register defines for HMR_core_regs
+
+#ifndef _HMR_CORE_REGS_REG_DEFS_
+#define _HMR_CORE_REGS_REG_DEFS_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// Register width
+#define HMR_CORE_REGS_PARAM_REG_WIDTH 32
+
+// Value to determine wich redundancy mode the core with that ID is in.
+#define HMR_CORE_REGS_CURRENT_MODE_REG_OFFSET 0x0
+#define HMR_CORE_REGS_CURRENT_MODE_INDEPENDENT_BIT 0
+#define HMR_CORE_REGS_CURRENT_MODE_DUAL_BIT 1
+#define HMR_CORE_REGS_CURRENT_MODE_TRIPLE_BIT 2
+
+// Mismatches of the core
+#define HMR_CORE_REGS_MISMATCHES_REG_OFFSET 0x4
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+#endif  // _HMR_CORE_REGS_REG_DEFS_
+// End generated register defines for HMR_core_regs
+
+// Generated register defines for HMR_dmr_regs
+
+#ifndef _HMR_DMR_REGS_REG_DEFS_
+#define _HMR_DMR_REGS_REG_DEFS_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// Register width
+#define HMR_DMR_REGS_PARAM_REG_WIDTH 32
+
+// DMR configuration enable.
+#define HMR_DMR_REGS_DMR_ENABLE_REG_OFFSET 0x0
+#define HMR_DMR_REGS_DMR_ENABLE_TMR_ENABLE_BIT 0
+
+// DMR configuration bits.
+#define HMR_DMR_REGS_DMR_CONFIG_REG_OFFSET 0x4
+#define HMR_DMR_REGS_DMR_CONFIG_TODO_BIT 0
+
+// Address for the last checkpoint.
+#define HMR_DMR_REGS_CHECKPOINT_ADDR_REG_OFFSET 0x8
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+#endif  // _HMR_DMR_REGS_REG_DEFS_
+// End generated register defines for HMR_dmr_regs
+
+// Generated register defines for HMR_tmr_regs
+
+#ifndef _HMR_TMR_REGS_REG_DEFS_
+#define _HMR_TMR_REGS_REG_DEFS_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// Register width
+#define HMR_TMR_REGS_PARAM_REG_WIDTH 32
+
+// TMR configuration enable.
+#define HMR_TMR_REGS_TMR_ENABLE_REG_OFFSET 0x0
+#define HMR_TMR_REGS_TMR_ENABLE_TMR_ENABLE_BIT 0
+
+// TMR configuration bits.
+#define HMR_TMR_REGS_TMR_CONFIG_REG_OFFSET 0x4
+#define HMR_TMR_REGS_TMR_CONFIG_DELAY_RESYNCH_BIT 0
+#define HMR_TMR_REGS_TMR_CONFIG_SETBACK_BIT 1
+#define HMR_TMR_REGS_TMR_CONFIG_RELOAD_SETBACK_BIT 2
+#define HMR_TMR_REGS_TMR_CONFIG_FORCE_RESYNCH_BIT 3
+
+// Stack Pointer storage register
+#define HMR_TMR_REGS_SP_STORE_REG_OFFSET 0x8
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+#endif  // _HMR_TMR_REGS_REG_DEFS_
+// End generated register defines for HMR_tmr_regs
+
+
+#endif // __ARCHI_HMR_HMR_V1_H__
diff --git a/include/hal/chips/pulp/pulp.h b/include/hal/chips/pulp/pulp.h
index 7450d08..0f14b49 100644
--- a/include/hal/chips/pulp/pulp.h
+++ b/include/hal/chips/pulp/pulp.h
@@ -47,4 +47,8 @@
 #include "hal/udma/spim/udma_spim_v3.h"
 #include "hal/udma/uart/udma_uart_v1.h"
 
+#ifdef ARCHI_HMR
+#include "hal/hmr/hmr_v1.h"
+#endif // ARCHI_HMR
+
 #endif
diff --git a/include/hal/hmr/hmr_v1.h b/include/hal/hmr/hmr_v1.h
new file mode 100644
index 0000000..53325db
--- /dev/null
+++ b/include/hal/hmr/hmr_v1.h
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2023 ETH Zurich and University of Bologna
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __HAL_HMR_HMR_V1_H__
+#define __HAL_HMR_HMR_V1_H__
+
+#include "archi/hmr/hmr_v1.h"
+#include "archi/pulp.h"
+
+#define NUM_TMR_GROUPS (ARCHI_CLUSTER_NB_PE/3)
+#define NUM_TMR_CORES  (ARCHI_CLUSTER_NB_PE-(ARCHI_CLUSTER_NB_PE%3))
+#define NUM_DMR_GROUPS (ARCHI_CLUSTER_NB_PE/2)
+#define NUM_DMR_CORES  (ARCHI_CLUSTER_NB_PE-(ARCHI_CLUSTER_NB_PE%2))
+
+// Interleaved cores
+#define TMR_IS_CORE(core_id)          (core_id<NUM_TMR_CORES)
+#if HMR_IN_INTERLEAVED
+#define TMR_GROUP_ID(core_id)         (core_id % NUM_TMR_GROUPS)
+#define TMR_CORE_ID(group_id, offset) (group_id + (offset * NUM_TMR_GROUPS))
+#define TMR_BARRIER_ID(group_id)      (1+group_id)
+#define TMR_BARRIER_SETUP(group_id)   (1<<group_id | 1<<(group_id+NUM_TMR_GROUPS) | 1<<(group_id+2*NUM_TMR_GROUPS))
+#else
+#define TMR_GROUP_ID(core_id)         (core_id/3)
+#define TMR_CORE_ID(group_id, offset) ((group_id * 3) + core_offset)
+#define TMR_BARRIER_ID(group_id)      (1+group_id+(group_id/2))
+#define TMR_BARRIER_SETUP(group_id)   (1<<(3*group_id) | 1<<(3*group_id + 1) | 1<<(3*group_id+2))
+#endif
+#define TMR_IS_MAIN_CORE(core_id)     (TMR_IS_CORE(core_id) && (TMR_CORE_ID(TMR_GROUP_ID(core_id), 0) == core_id))
+
+static inline unsigned int hmr_get_available_config(unsigned int cid) {
+  return pulp_read32(ARCHI_HMR_GLOBAL_ADDR(cid) + HMR_TOP_OFFSET + HMR_REGISTERS_AVAIL_CONFIG_REG_OFFSET);
+}
+
+static inline unsigned int hmr_get_active_cores(unsigned int cid) {
+  return pulp_read32(ARCHI_HMR_GLOBAL_ADDR(cid) + HMR_TOP_OFFSET + HMR_REGISTERS_CORES_EN_REG_OFFSET);
+}
+
+static inline unsigned int hmr_get_core_status(unsigned int cid, unsigned int core_id) {
+  return pulp_read32(ARCHI_HMR_GLOBAL_ADDR(cid) + HMR_CORE_OFFSET + (core_id * HMR_CORE_INCREMENT) + HMR_CORE_REGS_CURRENT_MODE_REG_OFFSET);
+}
+
+static inline unsigned int hmr_get_core_mismatches(unsigned int cid, unsigned int core_id) {
+  return pulp_read32(ARCHI_HMR_GLOBAL_ADDR(cid) + HMR_CORE_OFFSET + (core_id * HMR_CORE_INCREMENT) + HMR_CORE_REGS_MISMATCHES_REG_OFFSET);
+}
+
+static inline unsigned int hmr_reset_core_mismatches(unsigned int cid, unsigned int core_id) {
+  return pulp_write32(ARCHI_HMR_GLOBAL_ADDR(cid) + HMR_CORE_OFFSET + (core_id * HMR_CORE_INCREMENT) + HMR_CORE_REGS_MISMATCHES_REG_OFFSET, 0);
+}
+
+static inline unsigned int hmr_get_dmr_status_all(unsigned int cid) {
+  return pulp_read32(ARCHI_HMR_GLOBAL_ADDR(cid) + HMR_TOP_OFFSET + HMR_REGISTERS_DMR_ENABLE_REG_OFFSET);
+}
+
+static inline void hmr_set_dmr_status_all(unsigned int cid, unsigned int status) {
+  pulp_write32(ARCHI_HMR_GLOBAL_ADDR(cid) + HMR_TOP_OFFSET + HMR_REGISTERS_DMR_ENABLE_REG_OFFSET, status);
+}
+
+static inline void hmr_enable_all_dmr(unsigned int cid) {
+  hmr_set_dmr_status_all(cid, (1<<NUM_TMR_GROUPS)-1);
+}
+
+static inline void hmr_disable_all_dmr(unsigned int cid) {
+  hmr_set_dmr_status_all(cid, 0);
+}
+
+static inline unsigned int hmr_get_tmr_status_all(unsigned int cid) {
+  return pulp_read32(ARCHI_HMR_GLOBAL_ADDR(cid) + HMR_TOP_OFFSET + HMR_REGISTERS_TMR_ENABLE_REG_OFFSET);
+}
+
+static inline void hmr_set_tmr_status_all(unsigned int cid, unsigned int status) {
+  pulp_write32(ARCHI_HMR_GLOBAL_ADDR(cid) + HMR_TOP_OFFSET + HMR_REGISTERS_TMR_ENABLE_REG_OFFSET, status);
+}
+
+static inline void hmr_enable_all_tmr(unsigned int cid) {
+  hmr_set_tmr_status_all(cid, (1<<NUM_TMR_GROUPS)-1);
+}
+
+static inline void hmr_disable_all_tmr(unsigned int cid) {
+  hmr_set_tmr_status_all(cid, 0);
+}
+
+static void hmr_tmr_barrier_setup_all() {
+  for (int i = 0; i < NUM_TMR_GROUPS; i++) {
+    eu_bar_setup(eu_bar_addr(TMR_BARRIER_ID(i)), TMR_BARRIER_SETUP(i));
+  }
+}
+
+#endif // __HAL_HMR_HMR_V1_H__
diff --git a/kernel/cluster.c b/kernel/cluster.c
index 872eab9..2203b6a 100644
--- a/kernel/cluster.c
+++ b/kernel/cluster.c
@@ -40,7 +40,11 @@ static void cluster_core_init()
 {
     eu_evt_maskSet((1<<PULP_DISPATCH_EVENT) | (1<<PULP_MUTEX_EVENT) | (1<<PULP_HW_BAR_EVENT));
 
+#ifdef ARCHI_HMR
+    eu_bar_setup(eu_bar_addr(0), hmr_get_active_cores(0));
+#else
     eu_bar_setup(eu_bar_addr(0), (1<<ARCHI_CLUSTER_NB_PE) - 1);
+#endif
 }
 
 void cluster_entry_stub()
diff --git a/rules/pulpos/targets/pulp.mk b/rules/pulpos/targets/pulp.mk
index 1872a36..750bdd6 100644
--- a/rules/pulpos/targets/pulp.mk
+++ b/rules/pulpos/targets/pulp.mk
@@ -18,6 +18,10 @@ PULP_ARCH_LDFLAGS ?=  -march=rv32imcxgap9
 PULP_ARCH_OBJDFLAGS ?= -Mmarch=rv32imcxgap9
 endif
 
+ifdef ARCHI_HMR
+PULP_CFLAGS += -DARCHI_HMR
+endif
+
 PULP_CFLAGS    += -fdata-sections -ffunction-sections -include chips/pulp/config.h -I$(PULPRT_HOME)/include/chips/pulp
 PULP_OMP_CFLAGS    += -fopenmp -mnativeomp
 PULP_LDFLAGS += -nostartfiles -nostdlib -Wl,--gc-sections -L$(PULPRT_HOME)/kernel -Tchips/pulp/link.ld -lgcc