# HG changeset patch

# User gromero

# Date 1495057954 14400

#      Wed May 17 17:52:34 2017 -0400

# Node ID 74c81011375b5f432df155dcd7b3c9a668b45740

# Parent  4d9931ebf8617b1b06adbc1beee6ed1b58661a8b

8175813: PPC64: "mbind: Invalid argument" when -XX:+UseNUMA is used

diff --git a/src/os/linux/vm/os_linux.cpp b/src/os/linux/vm/os_linux.cpp

--- openjdk/hotspot/src/os/linux/vm/os_linux.cpp

+++ openjdk/hotspot/src/os/linux/vm/os_linux.cpp

@@ -2736,8 +2736,9 @@

 bool os::numa_topology_changed()   { return false; }

 size_t os::numa_get_groups_num() {

-  int max_node = Linux::numa_max_node();

-  return max_node > 0 ? max_node + 1 : 1;

+  // Return just the number of nodes in which it's possible to allocate memory

+  // (in numa terminology, configured nodes).

+  return Linux::numa_num_configured_nodes();

 }

 int os::numa_get_group_id() {

@@ -2751,11 +2752,33 @@

   return 0;

 }

+int os::Linux::get_existing_num_nodes() {

+  size_t node;

+  size_t highest_node_number = Linux::numa_max_node();

+  int num_nodes = 0;

+

+  // Get the total number of nodes in the system including nodes without memory.

+  for (node = 0; node <= highest_node_number; node++) {

+    if (isnode_in_existing_nodes(node)) {

+      num_nodes++;

+    }

+  }

+  return num_nodes;

+}

+

 size_t os::numa_get_leaf_groups(int *ids, size_t size) {

-  for (size_t i = 0; i < size; i++) {

-    ids[i] = i;

-  }

-  return size;

+  size_t highest_node_number = Linux::numa_max_node();

+  size_t i = 0;

+

+  // Map all node ids in which is possible to allocate memory. Also nodes are

+  // not always consecutively available, i.e. available from 0 to the highest

+  // node number.

+  for (size_t node = 0; node <= highest_node_number; node++) {

+    if (Linux::isnode_in_configured_nodes(node)) {

+      ids[i++] = node;

+    }

+  }

+  return i;

 }

 bool os::get_page_info(char *start, page_info* info) {

@@ -2825,18 +2848,28 @@

                                            libnuma_dlsym(handle, "numa_node_to_cpus")));

       set_numa_max_node(CAST_TO_FN_PTR(numa_max_node_func_t,

                                        libnuma_dlsym(handle, "numa_max_node")));

+      set_numa_num_configured_nodes(CAST_TO_FN_PTR(numa_num_configured_nodes_func_t,

+                                                   libnuma_dlsym(handle, "numa_num_configured_nodes")));

       set_numa_available(CAST_TO_FN_PTR(numa_available_func_t,

                                         libnuma_dlsym(handle, "numa_available")));

       set_numa_tonode_memory(CAST_TO_FN_PTR(numa_tonode_memory_func_t,

                                             libnuma_dlsym(handle, "numa_tonode_memory")));

       set_numa_interleave_memory(CAST_TO_FN_PTR(numa_interleave_memory_func_t,

-                                            libnuma_dlsym(handle, "numa_interleave_memory")));

+                                                libnuma_dlsym(handle, "numa_interleave_memory")));

       set_numa_set_bind_policy(CAST_TO_FN_PTR(numa_set_bind_policy_func_t,

-                                            libnuma_dlsym(handle, "numa_set_bind_policy")));

-

+                                              libnuma_dlsym(handle, "numa_set_bind_policy")));

+      set_numa_bitmask_isbitset(CAST_TO_FN_PTR(numa_bitmask_isbitset_func_t,

+                                               libnuma_dlsym(handle, "numa_bitmask_isbitset")));

+      set_numa_distance(CAST_TO_FN_PTR(numa_distance_func_t,

+                                       libnuma_dlsym(handle, "numa_distance")));

       if (numa_available() != -1) {

         set_numa_all_nodes((unsigned long*)libnuma_dlsym(handle, "numa_all_nodes"));

+        set_numa_all_nodes_ptr((struct bitmask **)libnuma_dlsym(handle, "numa_all_nodes_ptr"));

+        set_numa_nodes_ptr((struct bitmask **)libnuma_dlsym(handle, "numa_nodes_ptr"));

+        // Create an index -> node mapping, since nodes are not always consecutive

+        _nindex_to_node = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<int>(0, true);

+        rebuild_nindex_to_node_map();

         // Create a cpu -> node mapping

         _cpu_to_node = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<int>(0, true);

         rebuild_cpu_to_node_map();

@@ -2847,6 +2880,17 @@

   return false;

 }

+void os::Linux::rebuild_nindex_to_node_map() {

+  int highest_node_number = Linux::numa_max_node();

+

+  nindex_to_node()->clear();

+  for (int node = 0; node <= highest_node_number; node++) {

+    if (Linux::isnode_in_existing_nodes(node)) {

+      nindex_to_node()->append(node);

+    }

+  }

+}

+

 // rebuild_cpu_to_node_map() constructs a table mapping cpud id to node id.

 // The table is later used in get_node_by_cpu().

 void os::Linux::rebuild_cpu_to_node_map() {

@@ -2866,16 +2910,46 @@

   cpu_to_node()->clear();

   cpu_to_node()->at_grow(cpu_num - 1);

-  size_t node_num = numa_get_groups_num();

-

+

+  size_t node_num = get_existing_num_nodes();

+

+  int distance = 0;

+  int closest_distance = INT_MAX;

+  int closest_node = 0;

   unsigned long *cpu_map = NEW_C_HEAP_ARRAY(unsigned long, cpu_map_size, mtInternal);

   for (size_t i = 0; i < node_num; i++) {

-    if (numa_node_to_cpus(i, cpu_map, cpu_map_size * sizeof(unsigned long)) != -1) {

+    // Check if node is configured (not a memory-less node). If it is not, find

+    // the closest configured node.

+    if (!isnode_in_configured_nodes(nindex_to_node()->at(i))) {

+      closest_distance = INT_MAX;

+      // Check distance from all remaining nodes in the system. Ignore distance

+      // from itself and from another non-configured node.

+      for (size_t m = 0; m < node_num; m++) {

+        if (m != i && isnode_in_configured_nodes(nindex_to_node()->at(m))) {

+          distance = numa_distance(nindex_to_node()->at(i), nindex_to_node()->at(m));

+          // If a closest node is found, update. There is always at least one

+          // configured node in the system so there is always at least one node

+          // close.

+          if (distance != 0 && distance < closest_distance) {

+            closest_distance = distance;

+            closest_node = nindex_to_node()->at(m);

+          }

+        }

+      }

+     } else {

+       // Current node is already a configured node.

+       closest_node = nindex_to_node()->at(i);

+     }

+

+    // Get cpus from the original node and map them to the closest node. If node

+    // is a configured node (not a memory-less node), then original node and

+    // closest node are the same.

+    if (numa_node_to_cpus(nindex_to_node()->at(i), cpu_map, cpu_map_size * sizeof(unsigned long)) != -1) {

       for (size_t j = 0; j < cpu_map_valid_size; j++) {

         if (cpu_map[j] != 0) {

           for (size_t k = 0; k < BitsPerCLong; k++) {

             if (cpu_map[j] & (1UL << k)) {

-              cpu_to_node()->at_put(j * BitsPerCLong + k, i);

+              cpu_to_node()->at_put(j * BitsPerCLong + k, closest_node);

             }

           }

         }

@@ -2893,14 +2967,20 @@

 }

 GrowableArray<int>* os::Linux::_cpu_to_node;

+GrowableArray<int>* os::Linux::_nindex_to_node;

 os::Linux::sched_getcpu_func_t os::Linux::_sched_getcpu;

 os::Linux::numa_node_to_cpus_func_t os::Linux::_numa_node_to_cpus;

 os::Linux::numa_max_node_func_t os::Linux::_numa_max_node;

+os::Linux::numa_num_configured_nodes_func_t os::Linux::_numa_num_configured_nodes;

 os::Linux::numa_available_func_t os::Linux::_numa_available;

 os::Linux::numa_tonode_memory_func_t os::Linux::_numa_tonode_memory;

 os::Linux::numa_interleave_memory_func_t os::Linux::_numa_interleave_memory;

 os::Linux::numa_set_bind_policy_func_t os::Linux::_numa_set_bind_policy;

+os::Linux::numa_bitmask_isbitset_func_t os::Linux::_numa_bitmask_isbitset;

+os::Linux::numa_distance_func_t os::Linux::_numa_distance;

 unsigned long* os::Linux::_numa_all_nodes;

+struct bitmask* os::Linux::_numa_all_nodes_ptr;

+struct bitmask* os::Linux::_numa_nodes_ptr;

 bool os::pd_uncommit_memory(char* addr, size_t size) {

   uintptr_t res = (uintptr_t) ::mmap(addr, size, PROT_NONE,

diff --git a/src/os/linux/vm/os_linux.hpp b/src/os/linux/vm/os_linux.hpp

--- openjdk/hotspot/src/os/linux/vm/os_linux.hpp

+++ openjdk/hotspot/src/os/linux/vm/os_linux.hpp

@@ -67,6 +67,7 @@

   static bool _supports_fast_thread_cpu_time;

   static GrowableArray<int>* _cpu_to_node;

+  static GrowableArray<int>* _nindex_to_node;

  protected:

@@ -94,7 +95,9 @@

   static void set_is_floating_stack()         { _is_floating_stack = true; }

   static void rebuild_cpu_to_node_map();

+  static void rebuild_nindex_to_node_map();

   static GrowableArray<int>* cpu_to_node()    { return _cpu_to_node; }

+  static GrowableArray<int>* nindex_to_node()  { return _nindex_to_node; }

   static size_t find_large_page_size();

   static size_t setup_large_page_size();

@@ -243,28 +246,41 @@

   typedef int (*sched_getcpu_func_t)(void);

   typedef int (*numa_node_to_cpus_func_t)(int node, unsigned long *buffer, int bufferlen);

   typedef int (*numa_max_node_func_t)(void);

+  typedef int (*numa_num_configured_nodes_func_t)(void);

   typedef int (*numa_available_func_t)(void);

   typedef int (*numa_tonode_memory_func_t)(void *start, size_t size, int node);

   typedef void (*numa_interleave_memory_func_t)(void *start, size_t size, unsigned long *nodemask);

   typedef void (*numa_set_bind_policy_func_t)(int policy);

+  typedef int (*numa_bitmask_isbitset_func_t)(struct bitmask *bmp, unsigned int n);

+  typedef int (*numa_distance_func_t)(int node1, int node2);

   static sched_getcpu_func_t _sched_getcpu;

   static numa_node_to_cpus_func_t _numa_node_to_cpus;

   static numa_max_node_func_t _numa_max_node;

+  static numa_num_configured_nodes_func_t _numa_num_configured_nodes;

   static numa_available_func_t _numa_available;

   static numa_tonode_memory_func_t _numa_tonode_memory;

   static numa_interleave_memory_func_t _numa_interleave_memory;

   static numa_set_bind_policy_func_t _numa_set_bind_policy;

+  static numa_bitmask_isbitset_func_t _numa_bitmask_isbitset;

+  static numa_distance_func_t _numa_distance;

   static unsigned long* _numa_all_nodes;

+  static struct bitmask* _numa_all_nodes_ptr;

+  static struct bitmask* _numa_nodes_ptr;

   static void set_sched_getcpu(sched_getcpu_func_t func) { _sched_getcpu = func; }

   static void set_numa_node_to_cpus(numa_node_to_cpus_func_t func) { _numa_node_to_cpus = func; }

   static void set_numa_max_node(numa_max_node_func_t func) { _numa_max_node = func; }

+  static void set_numa_num_configured_nodes(numa_num_configured_nodes_func_t func) { _numa_num_configured_nodes = func; }

   static void set_numa_available(numa_available_func_t func) { _numa_available = func; }

   static void set_numa_tonode_memory(numa_tonode_memory_func_t func) { _numa_tonode_memory = func; }

   static void set_numa_interleave_memory(numa_interleave_memory_func_t func) { _numa_interleave_memory = func; }

   static void set_numa_set_bind_policy(numa_set_bind_policy_func_t func) { _numa_set_bind_policy = func; }

+  static void set_numa_bitmask_isbitset(numa_bitmask_isbitset_func_t func) { _numa_bitmask_isbitset = func; }

+  static void set_numa_distance(numa_distance_func_t func) { _numa_distance = func; }

   static void set_numa_all_nodes(unsigned long* ptr) { _numa_all_nodes = ptr; }

+  static void set_numa_all_nodes_ptr(struct bitmask **ptr) { _numa_all_nodes_ptr = *ptr; }

+  static void set_numa_nodes_ptr(struct bitmask **ptr) { _numa_nodes_ptr = *ptr; }

   static int sched_getcpu_syscall(void);

 public:

   static int sched_getcpu()  { return _sched_getcpu != NULL ? _sched_getcpu() : -1; }

@@ -272,6 +288,9 @@

     return _numa_node_to_cpus != NULL ? _numa_node_to_cpus(node, buffer, bufferlen) : -1;

   }

   static int numa_max_node() { return _numa_max_node != NULL ? _numa_max_node() : -1; }

+  static int numa_num_configured_nodes() {

+    return _numa_num_configured_nodes != NULL ? _numa_num_configured_nodes() : -1;

+  }

   static int numa_available() { return _numa_available != NULL ? _numa_available() : -1; }

   static int numa_tonode_memory(void *start, size_t size, int node) {

     return _numa_tonode_memory != NULL ? _numa_tonode_memory(start, size, node) : -1;

@@ -286,7 +305,25 @@

       _numa_set_bind_policy(policy);

     }

   }

+  static int numa_distance(int node1, int node2) {

+    return _numa_distance != NULL ? _numa_distance(node1, node2) : -1;

+  }

   static int get_node_by_cpu(int cpu_id);

+  static int get_existing_num_nodes();

+  // Check if numa node is configured (non-zero memory node).

+  static bool isnode_in_configured_nodes(unsigned int n) {

+    if (_numa_bitmask_isbitset != NULL && _numa_all_nodes_ptr != NULL) {

+      return _numa_bitmask_isbitset(_numa_all_nodes_ptr, n);

+    } else

+      return 0;

+  }

+  // Check if numa node exists in the system (including zero memory nodes).

+  static bool isnode_in_existing_nodes(unsigned int n) {

+    if (_numa_bitmask_isbitset != NULL && _numa_nodes_ptr != NULL) {

+      return _numa_bitmask_isbitset(_numa_nodes_ptr, n);

+    } else

+      return 0;

+  }

 };