Add ERST functional test case (V3)
this case is used to test read/write/clear operations
on ERST.
Pay attention, please use this case on the kernel >=2.6.39-rc1.
More detail information please refer the test case itself.
BTW, this case doesn't consider the situation such as duplicate
or missing id because current firmware has bugs. It will be
updated after the firmware fixes this issue.
V3 -> V2: Makefile without recursive make
V2 -> V1: add copyright information
Signed-off-by: Chen Gong <gong.chen@linux.intel.com>
Signed-off-by: Andi Kleen <ak@linux.intel.com>
diff --git a/tsrc/Makefile b/tsrc/Makefile
index 9e62be9..498cb10 100644
--- a/tsrc/Makefile
+++ b/tsrc/Makefile
@@ -5,7 +5,7 @@
KFLAGS := -I ./kinclude
-EXE := tinjpage tsimpleinj tkillpoison tprctl tsoft tsoftinj thugetlb
+EXE := tinjpage tsimpleinj tkillpoison tprctl tsoft tsoftinj thugetlb erst-inject
EXE += ttranshuge
EXEKERNEL := tring ttable
@@ -35,6 +35,8 @@
x.html: ttable
./ttable ${TFLAGS} > x.html
+include erst-inj/erst-inj.mk
+
.PHONY: see
see: x.html
@@ -69,5 +71,5 @@
test-kernel: tcases
./tcases
-
-
+test-erst: erst-inject
+ ./erst-inject.sh
diff --git a/tsrc/erst-inj/cper.h b/tsrc/erst-inj/cper.h
new file mode 100644
index 0000000..63cd94b
--- /dev/null
+++ b/tsrc/erst-inj/cper.h
@@ -0,0 +1,211 @@
+/*
+ * cper.h - UEFI Common Platform Error Record
+ *
+ * Copyright (C) 2009, Intel Corp.
+ * Author: Huang Ying <ying.huang@intel.com>
+ *
+ * This file is released under the GPLv2.
+ */
+
+#ifndef ACPI_CPER_H
+#define ACPI_CPER_H
+
+#include "uuid.h"
+
+#define CPER_SER_RECOVERABLE 0x0
+#define CPER_SER_FATAL 0x1
+#define CPER_SER_CORRECTED 0x2
+#define CPER_SER_INFORMATIONAL 0x3
+
+#define CPER_VALID_PLATFORM_ID 0x0001
+#define CPER_VALID_TIMESTAMP 0x0002
+#define CPER_VALID_PARTITION_ID 0x0004
+
+#define CPER_HW_ERROR_FLAGS_RECOVERED 0x1
+#define CPER_HW_ERROR_FLAGS_PREVERR 0x2
+#define CPER_HW_ERROR_FLAGS_SIMULATED 0x4
+
+#define CPER_SEC_VALID_FRU_ID 0x1
+#define CPER_SEC_VALID_FRU_STRING 0x2
+
+#define CPER_SEC_PRIMARY 0x0001
+#define CPER_SEC_CONTAINMENT_WARNING 0x0002
+#define CPER_SEC_RESET 0x0004
+#define CPER_SEC_ERROR_THRESHOLD_EXCEEDED 0x0008
+#define CPER_SEC_RESOURCE_NOT_ACCESSIBLE 0x0010
+#define CPER_SEC_LATENT_ERROR 0x0020
+
+#define CPER_SIG_RECORD "REPC"
+
+#define CPER_SIG_SIZE 4
+
+#define CPER_NOTIFY_CMC \
+ LGUID(0x2DCE8BB1, 0xBDD7, 0x450e, 0xB9, 0xAD, 0x9C, 0xF4, \
+ 0xEB, 0xD4, 0xF8, 0x90)
+#define CPER_NOTIFY_CPE \
+ LGUID(0x4E292F96, 0xD843, 0x4a55, 0xA8, 0xC2, 0xD4, 0x81, \
+ 0xF2, 0x7E, 0xBE, 0xEE)
+#define CPER_NOTIFY_MCE \
+ LGUID(0xE8F56FFE, 0x919C, 0x4cc5, 0xBA, 0x88, 0x65, 0xAB, \
+ 0xE1, 0x49, 0x13, 0xBB)
+#define CPER_NOTIFY_PCIE \
+ LGUID(0xCF93C01F, 0x1A16, 0x4dfc, 0xB8, 0xBC, 0x9C, 0x4D, \
+ 0xAF, 0x67, 0xC1, 0x04)
+#define CPER_NOTIFY_INIT \
+ LGUID(0xCC5263E8, 0x9308, 0x454a, 0x89, 0xD0, 0x34, 0x0B, \
+ 0xD3, 0x9B, 0xC9, 0x8E)
+#define CPER_NOTIFY_NMI \
+ LGUID(0x5BAD89FF, 0xB7E6, 0x42c9, 0x81, 0x4A, 0xCF, 0x24, \
+ 0x85, 0xD6, 0xE9, 0x8A)
+#define CPER_NOTIFY_BOOT \
+ LGUID(0x3D61A466, 0xAB40, 0x409a, 0xA6, 0x98, 0xF3, 0x62, \
+ 0xD4, 0x64, 0xB3, 0x8F)
+#define CPER_NOTIFY_DMAR \
+ LGUID(0x667DD791, 0xC6B3, 0x4c27, 0x8A, 0x6B, 0x0F, 0x8E, \
+ 0x72, 0x2D, 0xEB, 0x41)
+
+#define CPER_SEC_PROC_GENERIC \
+ LGUID(0x9876CCAD, 0x47B4, 0x4bdb, 0xB6, 0x5E, 0x16, 0xF1, \
+ 0x93, 0xC4, 0xF3, 0xDB)
+#define CPER_SEC_PROC_IA \
+ LGUID(0xDC3EA0B0, 0xA144, 0x4797, 0xB9, 0x5B, 0x53, 0xFA, \
+ 0x24, 0x2B, 0x6E, 0x1D)
+#define CPER_SEC_PROC_IPF \
+ LGUID(0xE429FAF1, 0x3CB7, 0x11D4, 0x0B, 0xCA, 0x07, 0x00, \
+ 0x80, 0xC7, 0x3C, 0x88, 0x81)
+#define CPER_SEC_PLATFORM_MEM \
+ LGUID(0xA5BC1114, 0x6F64, 0x4EDE, 0xB8, 0x63, 0x3E, 0x83, \
+ 0xED, 0x7C, 0x83, 0xB1)
+#define CPER_SEC_PCIE \
+ LGUID(0xD995E954, 0xBBC1, 0x430F, 0xAD, 0x91, 0xB4, 0x4D, \
+ 0xCB, 0x3C, 0x6F, 0x35)
+#define CPER_SEC_FW_ERR_REC_REF \
+ LGUID(0x81212A96, 0x09ED, 0x4996, 0x94, 0x71, 0x8D, 0x72, \
+ 0x9C, 0x8E, 0x69, 0xED)
+#define CPER_SEC_PCI_X_BUS \
+ LGUID(0xC5753963, 0x3B84, 0x4095, 0xBF, 0x78, 0xED, 0xDA, \
+ 0xD3, 0xF9, 0xC9, 0xDD)
+#define CPER_SEC_PCI_DEV \
+ LGUID(0xEB5E4685, 0xCA66, 0x4769, 0xB6, 0xA2, 0x26, 0x06, \
+ 0x8B, 0x00, 0x13, 0x26)
+#define CPER_SEC_DMAR_GENERIC \
+ LGUID(0x5B51FEF7, 0xC79D, 0x4434, 0x8F, 0x1B, 0xAA, 0x62, \
+ 0xDE, 0x3E, 0x2C, 0x64)
+#define CPER_SEC_DMAR_VT \
+ LGUID(0x71761D37, 0x32B2, 0x45cd, 0xA7, 0xD0, 0xB0, 0xFE, \
+ 0xDD, 0x93, 0xE8, 0xCF)
+#define CPER_SEC_DMAR_IOMMU \
+ LGUID(0x036F84E1, 0x7F37, 0x428c, 0xA7, 0x9E, 0x57, 0x5F, \
+ 0xDF, 0xAA, 0x84, 0xEC)
+
+/*
+ * All tables and structs must be byte-packed to match CPER
+ * specification, since the tables are provided by the system BIOS
+ */
+#pragma pack(1)
+
+struct cper_record_header
+{
+ char signature[CPER_SIG_SIZE]; /* must be "REPC" */
+ __u16 revision;
+ __u32 signature_end; /* must be 0xffffffff */
+ __u16 section_count;
+ __u32 error_severity;
+ __u32 validation_bits;
+ __u32 record_length;
+ __u64 timestamp;
+ lguid_t platform_id;
+ lguid_t partition_id;
+ lguid_t creator_id;
+ lguid_t notification_type;
+ __u64 record_id;
+ __u32 flags;
+ __u64 persistence_information;
+ __u8 reserved[12]; /* must be zero */
+};
+
+struct cper_section_descriptor
+{
+ __u32 section_offset; /* Offset in bytes of the
+ * section body from the base
+ * of the record header */
+ __u32 section_length;
+ __u16 revision;
+ __u8 validation_bits;
+ __u8 reserved; /* must be zero */
+ __u32 flags;
+ lguid_t section_type;
+ lguid_t fru_id;
+ __u32 section_severity;
+ __u8 fru_text[20];
+};
+
+struct cper_sec_proc_generic
+{
+ __u64 validation_bits;
+ __u8 proc_type;
+ __u8 proc_isa;
+ __u8 proc_error_type;
+ __u8 operation;
+ __u8 flags;
+ __u8 level;
+ __u16 reserved;
+ __u64 cpu_version;
+ char cpu_brand[128];
+ __u64 proc_id;
+ __u64 target_addr;
+ __u64 requestor_id;
+ __u64 responder_id;
+ __u64 ip;
+};
+
+struct cper_sec_proc_ia
+{
+ __u64 validation_bits;
+ __u8 lapic_id;
+ __u8 cpuid[48];
+};
+
+struct cper_ia_err_info
+{
+ lguid_t err_type;
+ __u64 validation_bits;
+ __u64 check_info;
+ __u64 target_id;
+ __u64 requestor_id;
+ __u64 responder_id;
+ __u64 ip;
+};
+
+struct cper_ia_proc_ctx
+{
+ __u16 reg_ctx_type;
+ __u16 reg_arr_size;
+ __u32 msr_addr;
+ __u64 mm_reg_addr;
+};
+
+struct cper_sec_mem_err
+{
+ __u64 validation_bits;
+ __u64 error_status;
+ __u64 physical_addr;
+ __u64 physical_addr_mask;
+ __u16 node;
+ __u16 card;
+ __u16 module;
+ __u16 bank;
+ __u16 device;
+ __u16 row;
+ __u16 column;
+ __u16 bit_pos;
+ __u64 requestor_id;
+ __u64 responder_id;
+ __u64 target_id;
+ __u8 error_type;
+};
+
+/* Reset to default packing */
+#pragma pack()
+
+#endif
diff --git a/tsrc/erst-inj/erst-inj.mk b/tsrc/erst-inj/erst-inj.mk
new file mode 100644
index 0000000..32f253c
--- /dev/null
+++ b/tsrc/erst-inj/erst-inj.mk
@@ -0,0 +1,4 @@
+CFLAGS := -g -Wall
+
+erst-inject: erst-inj/erst-inject.c
+ ${CC} ${CFLAGS} -o erst-inject erst-inj/erst-inject.c
diff --git a/tsrc/erst-inj/erst-inject.c b/tsrc/erst-inj/erst-inject.c
new file mode 100644
index 0000000..3610f61
--- /dev/null
+++ b/tsrc/erst-inj/erst-inject.c
@@ -0,0 +1,251 @@
+/*
+ * Error Record Serialization Table(ERST) is used to save and retrieve hardware
+ * error information to and from a persistent store, such as flash or NVRAM.
+ *
+ * This test case is used to test ERST operation including read/write/clean.
+ * To be sure of loading erst-dbg module before executing this test.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should find a copy of v2 of the GNU General Public License somewhere
+ * on your Linux system; if not, write to the Free Software Foundation,
+ * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ * Copyright (C) 2011, Intel Corp.
+ * Author: Chen Gong <gong.chen@intel.com>
+ *
+ * Original written by Huang Ying <ying.huang@intel.com>
+ * Updated by Chen Gong <gong.chen@intel.com>
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/ioctl.h>
+
+#include "cper.h"
+
+#define ERST_DEV "/dev/erst_dbg"
+
+#define APEI_ERST_CLEAR_RECORD _IOW('E', 1, u64)
+#define APEI_ERST_GET_RECORD_COUNT _IOR('E', 2, u32)
+
+#define CPER_CREATOR_LINUX \
+ LGUID(0x94DB0E05, 0xEE60, 0x42D8, 0x91, 0xA5, 0xC6, 0xC0, \
+ 0x02, 0x41, 0x6C, 0x6A)
+
+#define ERROR_EXIT_ON(check, fmt, x...) \
+ do { \
+ if (check) \
+ error_exit(fmt, ## x); \
+ } while (0)
+
+void error_exit(char *fmt, ...)
+{
+ va_list ap;
+
+ fprintf(stderr, "Error: ");
+ va_start(ap, fmt);
+ vfprintf(stderr, fmt, ap);
+ va_end(ap);
+
+ if (errno)
+ fprintf(stderr, ", errno: %d (%s)\n", errno, strerror(errno));
+ else
+ fprintf(stderr, "\n");
+ exit(-1);
+}
+
+void inject(int fd, u64 record_id)
+{
+ int rc;
+ unsigned int len;
+ struct cper_record_header *rcd_hdr;
+ struct cper_section_descriptor *sec_hdr;
+ struct cper_sec_mem_err *mem_err;
+
+ len = sizeof(*rcd_hdr) + sizeof(*sec_hdr) + sizeof(*mem_err);
+ printf("sizes: %lu, %lu, %lu\n", sizeof(*rcd_hdr), sizeof(*sec_hdr),
+ sizeof(*mem_err));
+ rcd_hdr = malloc(len);
+ ERROR_EXIT_ON(!rcd_hdr, "Can not alloc mem");
+
+#define LE 0
+
+ sec_hdr = (void *)(rcd_hdr + 1);
+ mem_err = (void *)(sec_hdr + 1);
+
+ memset(rcd_hdr, 0, sizeof(*rcd_hdr));
+#if 0
+ memcpy(rcd_hdr->signature, "REPC", 4);
+#else
+ memcpy(rcd_hdr->signature, "CPER", 4);
+#endif
+ rcd_hdr->revision = 0x0100;
+ rcd_hdr->signature_end = 0xffffffff;
+ rcd_hdr->error_severity = CPER_SER_FATAL;
+ rcd_hdr->validation_bits = 0;
+ rcd_hdr->creator_id = CPER_CREATOR_LINUX;
+ rcd_hdr->notification_type = CPER_NOTIFY_NMI;
+ rcd_hdr->section_count = 1;
+ rcd_hdr->record_length = len;
+ rcd_hdr->record_id = record_id;
+#if LE
+ memcpy(&rcd_hdr->persistence_information, "RE", 2);
+#else
+ memcpy(&rcd_hdr->persistence_information, "ER", 2);
+#endif
+
+ memset(sec_hdr, 0, sizeof(*sec_hdr));
+ sec_hdr->section_offset = (void *)mem_err - (void *)rcd_hdr;
+ sec_hdr->section_length = sizeof(*mem_err);
+ sec_hdr->revision = 0x0100;
+ sec_hdr->validation_bits = 0;
+ sec_hdr->flags = 0;
+ sec_hdr->section_type = CPER_SEC_PLATFORM_MEM;
+ sec_hdr->section_severity = CPER_SER_FATAL;
+
+ memset(mem_err, 0, sizeof(*mem_err));
+ mem_err->validation_bits = 0x6;
+ mem_err->physical_addr = 0x2000;
+ mem_err->physical_addr_mask = ~0xfffULL;
+
+ rc = write(fd, rcd_hdr, len);
+ ERROR_EXIT_ON(rc != len, "Error inject: %d", rc);
+
+ free(rcd_hdr);
+}
+
+#define POLL_BUF_SIZ (1024 * 1024)
+
+int poll(int fd)
+{
+ int rc;
+ struct cper_record_header *rcd_hdr;
+ struct cper_section_descriptor *sec_hdr;
+ struct cper_sec_mem_err *mem_err;
+
+ rcd_hdr = malloc(POLL_BUF_SIZ);
+ ERROR_EXIT_ON(!rcd_hdr, "Can not alloc mem");
+
+ rc = read(fd, rcd_hdr, POLL_BUF_SIZ);
+ ERROR_EXIT_ON(rc < 0, "Error poll: %d", rc);
+
+ sec_hdr = (void *)(rcd_hdr + 1);
+ mem_err = (void *)(sec_hdr + 1);
+
+ printf("rc: %d\n", rc);
+
+ printf("rcd sig: %4s\n", rcd_hdr->signature);
+ printf("rcd id: 0x%llx\n", rcd_hdr->record_id);
+
+ free(rcd_hdr);
+
+ return rc;
+}
+
+void clear(int fd, u64 record_id)
+{
+ int rc;
+
+ printf("clear an error record: id = 0x%llx\n", record_id);
+
+ rc = ioctl(fd, APEI_ERST_CLEAR_RECORD, &record_id);
+ ERROR_EXIT_ON(rc, "Error clear: %d", rc);
+}
+
+void get_record_count(int fd, u32 *record_count)
+{
+ int rc;
+ rc = ioctl(fd, APEI_ERST_GET_RECORD_COUNT, record_count);
+ ERROR_EXIT_ON(rc, "Error get record count: %d", rc);
+
+ printf("total error record count: %u\n", *record_count);
+}
+
+enum {
+ ERST_INJECT,
+ ERST_POLL,
+ ERST_CLEAR,
+ ERST_COUNT,
+};
+
+void usage()
+{
+ printf("Usage: ./erst-inject [option] <id>\n");
+ printf("PAY ATTENTION, <id> is hexadecimal.\n");
+ printf("\tp\treturn all error records in the ERST\n");
+ printf("\ti\twrite an error record to be persisted into one item with <id>\n");
+ printf("\tc\tclean specific error record with <id>\n");
+ printf("\tn\treturn error records count in the ERST\n");
+ printf("\nExample:\t ./erst-inject -p\n");
+ printf("\t\t ./erst-inject -i 0x1234567\n");
+ printf("\t\t ./erst-inject -c 5050\n");
+ printf("\t\t ./erst-inject -n\n");
+}
+
+int main(int argc, char *argv[])
+{
+ int fd;
+ int todo = ERST_COUNT;
+ int opt;
+ u64 record_id = 0x12345678;
+ u32 record_count;
+
+ if (argc == 1) {
+ usage();
+ exit(0);
+ }
+
+ while ((opt = getopt(argc, argv, "pi:c:n")) != -1) {
+ switch (opt) {
+ case 'p':
+ todo = ERST_POLL;
+ break;
+ case 'i':
+ todo = ERST_INJECT;
+ record_id = strtoull(optarg, NULL, 16);
+ break;
+ case 'c':
+ todo = ERST_CLEAR;
+ record_id = strtoull(optarg, NULL, 16);
+ break;
+ case 'n':
+ todo = ERST_COUNT;
+ break;
+ }
+ }
+
+ fd = open(ERST_DEV, O_RDWR);
+ ERROR_EXIT_ON(fd < 0, "Can not open dev file");
+
+ switch (todo) {
+ case ERST_INJECT:
+ inject(fd, record_id);
+ break;
+ case ERST_POLL:
+ while (poll(fd));
+ break;
+ case ERST_CLEAR:
+ clear(fd, record_id);
+ break;
+ case ERST_COUNT:
+ get_record_count(fd, &record_count);
+ break;
+ }
+
+ close(fd);
+
+ return 0;
+}
diff --git a/tsrc/erst-inj/uuid.h b/tsrc/erst-inj/uuid.h
new file mode 100644
index 0000000..765656a
--- /dev/null
+++ b/tsrc/erst-inj/uuid.h
@@ -0,0 +1,28 @@
+#ifndef _LINUX_GUID_H_
+#define _LINUX_GUID_H_
+
+#include <string.h>
+#include <stdio.h>
+
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned long long u64;
+
+typedef unsigned char __u8;
+typedef unsigned short __u16;
+typedef unsigned int __u32;
+typedef unsigned long long __u64;
+
+typedef struct {
+ u8 b[16];
+} lguid_t;
+
+#define LGUID(a, b, c, d0, d1, d2, d3, d4, d5, d6, d7) \
+((lguid_t) \
+{{ (a) & 0xff, ((a) >> 8) & 0xff, ((a) >> 16) & 0xff, ((a) >> 24) & 0xff, \
+ (b) & 0xff, ((b) >> 8) & 0xff, \
+ (c) & 0xff, ((c) >> 8) & 0xff, \
+ (d0), (d1), (d2), (d3), (d4), (d5), (d6), (d7) }})
+
+#endif
diff --git a/tsrc/erst-inject.sh b/tsrc/erst-inject.sh
new file mode 100644
index 0000000..c22ff7a
--- /dev/null
+++ b/tsrc/erst-inject.sh
@@ -0,0 +1,108 @@
+#!/bin/bash
+
+# APEI ERST firmware interface and implementation has no multiple users
+# in mind. For example, there is four records in storage with ID: 1, 2,
+# 3 and 4, if two ERST readers enumerate the records via
+# GET_NEXT_RECORD_ID as follow,
+#
+# reader 1 reader 2
+# 1
+# 2
+# 3
+# 4
+# -1
+# -1
+#
+# where -1 signals there is no more record ID.
+#
+# Reader 1 has no chance to check record 2 and 4, while reader 2 has no
+# chance to check record 1 and 3. And any other GET_NEXT_RECORD_ID will
+# return -1, that is, other readers will has no chance to check any
+# record even they are not cleared by anyone.
+#
+# This makes raw GET_NEXT_RECORD_ID not suitable for usage of multiple
+# users.
+#
+# This issue has been resolved since 2.6.39-rc1, so please run this case
+# with Linux kernel >=2.6.39-rc1
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should find a copy of v2 of the GNU General Public License somewhere
+# on your Linux system; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+# Copyright (C) 2011, Intel Corp.
+# Author: Chen Gong <gong.chen@intel.com>
+#
+
+
+ID=0xdeadbeaf
+ERST=./erst-inj/erst-inject
+LOG=./erst.log
+MODSTATUS=0
+
+err()
+{
+ echo "$*"
+ echo "test fails"
+ exit 1
+}
+
+#prepare the test env
+ls /dev/erst_dbg >/dev/null 2>&1
+if [ ! $? -eq 0 ]; then
+ modinfo erst_dbg > /dev/null 2>&1
+ [ $? -eq 0 ] || err "please ensure module erst_dbg existing"
+ modprobe erst_dbg
+ [ $? -eq 0 ] || err "fail to load module erst_dbg"
+ MODSTATUS=1
+fi
+
+ls $ERST > /dev/null 2>&1
+[ $? -eq 0 ] || err "please compile the test program first"
+
+echo "write one error record into ERST..."
+$ERST -i $ID 1>/dev/null
+if [ ! $? -eq 0 ]; then
+ err "ERST writing operation fails"
+fi
+echo "done"
+# read all error records in ERST
+$ERST -p > $LOG
+echo "check if existing the error record written before..."
+grep -q $ID $LOG
+if [ ! $? -eq 0 ]; then
+ err "don't find the error record written before in ERST"
+fi
+echo "done"
+
+echo "clear the error record written before..."
+$ERST -c $ID 1>/dev/null
+if [ ! $? -eq 0 ]; then
+ err "ERST writing opertion fails"
+fi
+echo "done"
+
+#read all error records again
+$ERST -p > $LOG
+
+echo "check if the error record has been cleared..."
+grep -q $ID $LOG
+if [ $? -eq 0 ]; then
+ err "ERST clearing opertion fails"
+fi
+echo "done"
+echo -e "\ntest passes"
+
+rm -f $LOG
+if [ $MODSTATUS -eq 1 ]; then
+ rmmod -f erst_dbg
+fi
