2018년 1월 15일 월요일

mycroft picroft usb audio 장치 사용하기

다음의 두가지 설정을 하여 USB audio 를 기본output 장치로 사용할 수 있다.

pi@picroft:~/skills $ cat /etc/asound.conf
pcm.!default {
        type hw
        card 0
}

ctl.!default {
        type hw
        card 0
}

pi@picroft:~/skills $ cat /etc/mycroft/mycroft.conf
{
   "play_wav_cmdline": "aplay -Dplughw:1,0 %1",
   "play_mp3_cmdline": "mpg123 -a hw:0,0 %1",
   "enclosure": {
      "platform": "picroft",
      "update": true,
      "test": false
   },
   "tts": {
      "module": "mimic",
      "mimic": {
         "voice": "ap",
         "path": "/usr/local/bin/mimic"
      }
   },
   "ipc_path": "/ramdisk/mycroft/ipc/"
 }
작성자: 시간: 댓글 없음:

2017년 11월 8일 수요일

Raspberry Pi B+ rtl8188eu 사용.

raspberry pi  B+ 에 wifi usb dongle rtl8188eu 를 사용하는 환경에서 wifi가 접속되지 않는 상태

 lsusb 에서도 잘 잡히고
root@dpi:/home/pi# lsusb
Bus 001 Device 007: ID 15d9:0a4e Trust International B.V. 
Bus 001 Device 006: ID 413c:2106 Dell Computer Corp. Dell QuietKey Keyboard
Bus 001 Device 005: ID 1a40:0101 Terminus Technology Inc. Hub
Bus 001 Device 004: ID 0bda:8179 Realtek Semiconductor Corp. RTL8188EUS 802.11n Wireless Network Adapter
Bus 001 Device 003: ID 0424:ec00 Standard Microsystems Corp. SMSC9512/9514 Fast Ethernet Adapter
Bus 001 Device 002: ID 0424:9512 Standard Microsystems Corp. SMC9512/9514 USB Hub
Bus 001 Device 001: ID 1d6b:0002 Linux Foundation 2.0 root hub

lsmod 에서도 잘 잡힌다.
root@dpi:/home/pi# lsmod
Module                  Size  Used by
evdev                  11746  0
r8188eu               425640  0
cfg80211              527100  0 r8188eu   <<<<< used 가 0 임.
rfkill                 21373  3 cfg80211
snd_bcm2835            23131  0
snd_pcm                97825  1 snd_bcm2835
snd_timer              22706  1 snd_pcm
snd                    68784  3 snd_timer,snd_bcm2835,snd_pcm
bcm2835_gpiomem         3791  0
uio_pdrv_genirq         3718  0
uio                    10166  1 uio_pdrv_genirq
fixed                   3029  0
ip_tables              12512  0
x_tables               20921  1 ip_tables
ipv6                  384613  0

blacklist 등록도 하고
root@dpi:/home/pi# cat /etc/modprobe.d/8188eu.conf 
# r8188eu is staging, 8188eu is off-kernel
#blacklist brcmfmac
blacklist rtl8188eu
options rtl8188eu rtw_power_mgnt=0 rtw_enusbss=0



여러 방법을 써 봤지만, 아래의 방법이 최후의 성공 방법.

 root@dpi:/home/pi# cd /lib/systemd/system/
root@dpi:/home/pi# cp wpa_supplicant@.service wpa_supplicant_wlan0.service
root@dpi:/home/pi# systemctl enable wpa_supplicant_wlan0.service
root@dpi:/home/pi# reboot
재부팅 후 

 [  OK  ] Listening on Load/Save RF Kill Switch Status /dev/rfkill Watch.
[  OK  ] Started Hostname Service.
[  OK  ] Found device RTL8188EUS 802.11n Wireless Network Adapter.
[  OK  ] Started ifup for wlan0.
[  OK  ] Started dhcpcd on all interfaces.
[  OK  ] Reached target Network.
         Starting Permit User Sessions...
         Starting The Apache HTTP Server...
         Starting OpenBSD Secure Shell server...
         Starting /etc/rc.local Compatibility...
[  OK  ] Started Permit User Sessions.
My IP address is 192.192.1.19              <<< IP주소를 가져왔다.
[  OK  ] Started /etc/rc.local Compatibility.
         Starting Terminate Plymouth Boot Screen...

         Starting Hold until boot process finishes up...

root@dpi:/home/pi# lsmod
Module                  Size  Used by
evdev                  11746  0
r8188eu               425640  0
cfg80211              527100  1 r8188eu   <<<<< used 가 1 임.
rfkill                 21373  3 cfg80211
snd_bcm2835            23131  0
snd_pcm                97825  1 snd_bcm2835
snd_timer              22706  1 snd_pcm
snd                    68784  3 snd_timer,snd_bcm2835,snd_pcm
bcm2835_gpiomem         3791  0
uio_pdrv_genirq         3718  0
uio                    10166  1 uio_pdrv_genirq
fixed                   3029  0
ip_tables              12512  0
x_tables               20921  1 ip_tables

ipv6                  384613  0

작성자: 시간: 댓글 없음:

2017년 10월 27일 금요일

Raspberry Pi WiFi power mode 변경

WiFi USB 동글을 사용중 Power saving 모드 때문에 일정시간이 지나면 WiFi 통신이 되지 않는경우. 
Power saving 모드를 비활성화 시켜 통신이 지속되도록 만드는 절차.

 USB dongle : RealTek 8188CUS

 root@doorpi:/etc/network# lsusb
Bus 001 Device 004: ID 0bda:8176 Realtek Semiconductor Corp. RTL8188CUS 802.11n WLAN Adapter
Bus 001 Device 003: ID 0424:ec00 Standard Microsystems Corp. SMSC9512/9514 Fast Ethernet Adapter
Bus 001 Device 002: ID 0424:9512 Standard Microsystems Corp. SMC9512/9514 USB Hub
Bus 001 Device 001: ID 1d6b:0002 Linux Foundation 2.0 root hub

root@doorpi:/home/pi# cat /sys/module/8192cu/parameters/rtw_power_mgnt
1

1은 Power saving 모드가 활성화 된 상태, 0으로 바꾼다.

root@doorpi:/home/pi# vi /etc/modprobe.d/brtl8192cu.conf 

options 8192cu rtw_power_mgnt=0 rtw_enusbss=0


root@doorpi:/home/pi# reboot


root@doorpi:/home/pi# cat /sys/module/8192cu/parameters/rtw_power_mgnt
0
작성자: 시간: 댓글 없음:

2017년 2월 6일 월요일

rabbitmq esp8266 mqtt -> amqp

ESP8266 을 사용하여 수집된 데이터를 rabbitmq 에 전송하고, 
AQMP 를 사용하여 데이터를 수집하는 코드 샘플.




receive.py

 #!/usr/bin/env python
import pika

 credentials = pika.PlainCredentials('test', 'testpass')

 parameters = pika.ConnectionParameters('192.168.1.28',
                                   5672,
                                   '/energy',
                                   credentials)

 connection = pika.BlockingConnection(parameters)
channel = connection.channel()

 channel.exchange_declare(exchange='amq.topic', type='topic', durable=True)

 result=channel.queue_declare(exclusive=True)

 queue_name=result.method.queue

 channel.queue_bind(exchange='amq.topic', 
                      queue=queue_name,
                      routing_key="outTopic")

 def callback(ch, method, properties, body):
    print(" [x] Received %r" % body)

 channel.basic_consume(callback,
                      queue=queue_name,
                      no_ack=True)

 print(' [*] Waiting for messages. To exit press CTRL+C')
channel.start_consuming()


ESP12-MQTT-example01.ino 

/*
 Basic ESP8266 MQTT example

 This sketch demonstrates the capabilities of the pubsub library in combination
 with the ESP8266 board/library.

 It connects to an MQTT server then:
  - publishes "hello world" to the topic "outTopic" every two seconds
  - subscribes to the topic "inTopic", printing out any messages
    it receives. NB - it assumes the received payloads are strings not binary
  - If the first character of the topic "inTopic" is an 1, switch ON the ESP Led,
    else switch it off

 It will reconnect to the server if the connection is lost using a blocking
 reconnect function. See the 'mqtt_reconnect_nonblocking' example for how to
 achieve the same result without blocking the main loop.

 To install the ESP8266 board, (using Arduino 1.6.4+):
  - Add the following 3rd party board manager under "File -> Preferences -> Additional Boards Manager URLs":
       http://arduino.esp8266.com/stable/package_esp8266com_index.json
  - Open the "Tools -> Board -> Board Manager" and click install for the ESP8266"
  - Select your ESP8266 in "Tools -> Board"

*/

#include <ESP8266WiFi.h>
#include <PubSubClient.h>

// Update these with values suitable for your network.

const char* ssid = "YOUR_SSID";
const char* password = "AP_PASSWORD";
const char* mqtt_server = "192.168.1.28";

WiFiClient espClient;
PubSubClient client(espClient);
long lastMsg = 0;
char msg[50];
int value = 0;

void setup() {
  pinMode(BUILTIN_LED, OUTPUT);     // Initialize the BUILTIN_LED pin as an output
  Serial.begin(115200);
  setup_wifi();
  client.setServer(mqtt_server, 1883);
  client.setCallback(callback);
}

void setup_wifi() {

  delay(10);
  // We start by connecting to a WiFi network
  Serial.println();
  Serial.print("Connecting to ");
  Serial.println(ssid);

  WiFi.begin(ssid, password);

  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }

  Serial.println("");
  Serial.println("WiFi connected");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());
}

void callback(char* topic, byte* payload, unsigned int length) {
  Serial.print("Message arrived [");
  Serial.print(topic);
  Serial.print("] ");
  for (int i = 0; i < length; i++) {
    Serial.print((char)payload[i]);
  }
  Serial.println();

  // Switch on the LED if an 1 was received as first character
  if ((char)payload[0] == '1') {
    digitalWrite(BUILTIN_LED, LOW);   // Turn the LED on (Note that LOW is the voltage level
    // but actually the LED is on; this is because
    // it is acive low on the ESP-01)
  } else {
    digitalWrite(BUILTIN_LED, HIGH);  // Turn the LED off by making the voltage HIGH
  }

}

void reconnect() {
  // Loop until we're reconnected
  while (!client.connected()) {
    Serial.print("Attempting MQTT connection...");
    // Attempt to connect
    if (client.connect("ESP8266Client", "test", "testpass")) {
      Serial.println("connected");
      // Once connected, publish an announcement...
      client.publish("outTopic", "hello world");
      // ... and resubscribe
      client.subscribe("inTopic");
    } else {
      Serial.print("failed, rc=");
      Serial.print(client.state());
      Serial.println(" try again in 5 seconds");
      // Wait 5 seconds before retrying
      delay(5000);
    }
  }
}
void loop() {

  if (!client.connected()) {
    reconnect();
  }
  client.loop();

  long now = millis();
  if (now - lastMsg > 2000) {
    lastMsg = now;
    ++value;
    snprintf (msg, 75, "hello world #%ld", value);
    Serial.print("Publish message: ");
    Serial.println(msg);
    client.publish("outTopic", msg);
  }
}
작성자: 시간: 댓글 없음:

2017년 2월 5일 일요일

rabbitmq ipv4 서비스

rabbitmq 서비스 재시작.

pi@rabbit-pi:/etc/rabbitmq $ sudo systemctl restart rabbitmq-server


최초 실행 시 rabbitmq 는 ipv6 에서 5672 서비스 가능.
pi@rabbit-pi:~/test $ netstat -an|more
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State      
tcp        0      0 0.0.0.0:25672           0.0.0.0:*               LISTEN     
tcp        0      0 0.0.0.0:4369            0.0.0.0:*               LISTEN     
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN     
tcp        0      0 0.0.0.0:15672           0.0.0.0:*               LISTEN     
tcp        0      0 127.0.0.1:6010          0.0.0.0:*               LISTEN     
tcp        0      0 127.0.0.1:6011          0.0.0.0:*               LISTEN     
tcp        0      0 192.168.1.28:22         192.168.1.13:42954      ESTABLISHED
tcp        0      0 192.168.1.28:22         192.168.1.13:43512      ESTABLISHED
tcp        0      0 192.168.1.28:22         192.168.1.13:43500      ESTABLISHED
tcp        0      0 127.0.0.1:40659         127.0.0.1:4369          ESTABLISHED
tcp        0      0 127.0.0.1:4369          127.0.0.1:40659         ESTABLISHED
tcp        0      0 127.0.1.1:4369          127.0.0.1:41022         TIME_WAIT  
tcp6       0      0 :::5672                 :::*                    LISTEN     
tcp6       0      0 :::22                   :::*                    LISTEN     
tcp6       0      0 ::1:6010                :::*                    LISTEN     
tcp6       0      0 ::1:6011                :::*                    LISTEN     
tcp6       0      0 ::1:57068               ::1:5672                ESTABLISHED
tcp6       0      0 ::1:5672                ::1:57068               ESTABLISHED
tcp6       0      0 ::1:57962               ::1:6011                TIME_WAIT  
udp        0      0 0.0.0.0:68              0.0.0.0:*                          
udp        0      0 192.168.1.28:123        0.0.0.0:*                          
udp        0      0 127.0.0.1:123           0.0.0.0:*                          
udp        0      0 0.0.0.0:123             0.0.0.0:*                          
udp        0      0 0.0.0.0:50330           0.0.0.0:*                          
udp        0      0 0.0.0.0:5353            0.0.0.0:*                          
udp6       0      0 :::34779                :::*                               
udp6       0      0 fe80::8aa2:e9e0:b44:123 :::*                               
udp6       0      0 ::1:123                 :::*                               
udp6       0      0 :::123                  :::*                               
udp6       0      0 :::5353                 :::*                               
raw6       0      0 :::58                   :::*                    7          

ipv4 서비스 하기 위해 /etc/rabbitmq/rabbitmq.config 파일 생성(최초 설치 시 없는파일 이므로 만들어 사용).

pi@rabbit-pi:/etc/rabbitmq $ cat /etc/rabbitmq/rabbitmq.config 
[
  {rabbit, [
    {tcp_listeners, [{"0.0.0.0", 5672},
                     {"::1",       5672}]}
  ]}
].

서비스 restart
pi@rabbit-pi:/etc/rabbitmq $ sudo systemctl restart rabbitmq-server


network 서비스 확인

pi@rabbit-pi:/etc/rabbitmq $ netstat -an|more
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State      
tcp        0      0 0.0.0.0:5672            0.0.0.0:*               LISTEN     
tcp        0      0 0.0.0.0:25672           0.0.0.0:*               LISTEN     
tcp        0      0 0.0.0.0:4369            0.0.0.0:*               LISTEN     
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN     
tcp        0      0 0.0.0.0:15672           0.0.0.0:*               LISTEN     
tcp        0      0 127.0.0.1:6010          0.0.0.0:*               LISTEN     
tcp        0      0 127.0.0.1:6011          0.0.0.0:*               LISTEN     
tcp        0      0 127.0.0.1:43999         127.0.1.1:25672         TIME_WAIT  
tcp        0      0 127.0.0.1:4369          127.0.0.1:46258         TIME_WAIT  
tcp        0      0 127.0.0.1:33074         127.0.0.1:4369          ESTABLISHED
tcp        0      0 192.168.1.28:22         192.168.1.13:42954      ESTABLISHED
tcp        0      0 127.0.0.1:44711         127.0.0.1:4369          TIME_WAIT  
tcp        0      0 127.0.0.1:4369          127.0.0.1:33074         ESTABLISHED
tcp        0      0 192.168.1.28:22         192.168.1.13:43512      ESTABLISHED
tcp        0      0 127.0.1.1:4369          127.0.0.1:35267         TIME_WAIT  
tcp        0      0 192.168.1.28:22         192.168.1.13:43500      ESTABLISHED
tcp        0      0 127.0.1.1:4369          127.0.0.1:53111         TIME_WAIT  
tcp        0      0 127.0.0.1:35131         127.0.0.1:4369          TIME_WAIT  
tcp        0      0 127.0.1.1:4369          127.0.0.1:42002         TIME_WAIT  
tcp        0      0 127.0.0.1:60629         127.0.0.1:4369          TIME_WAIT  
tcp        0      0 127.0.1.1:4369          127.0.0.1:35784         TIME_WAIT  
tcp6       0      0 ::1:5672                :::*                    LISTEN     
tcp6       0      0 :::22                   :::*                    LISTEN     
tcp6       0      0 ::1:6010                :::*                    LISTEN     
tcp6       0      0 ::1:6011                :::*                    LISTEN     
udp        0      0 0.0.0.0:68              0.0.0.0:*                          
udp        0      0 192.168.1.28:123        0.0.0.0:*                          
udp        0      0 127.0.0.1:123           0.0.0.0:*                          
udp        0      0 0.0.0.0:123             0.0.0.0:*                          
udp        0      0 0.0.0.0:50330           0.0.0.0:*                          
udp        0      0 0.0.0.0:5353            0.0.0.0:*                          
udp6       0      0 :::34779                :::*                               
udp6       0      0 fe80::8aa2:e9e0:b44:123 :::*                               
udp6       0      0 ::1:123                 :::*                               
udp6       0      0 :::123                  :::*                               
udp6       0      0 :::5353                 :::*                               
raw6       0      0 :::58                   :::*                    7          



작성자: 시간: 댓글 없음:

2016년 10월 3일 월요일

redhat ceph 1.3 install

subscription-manager repos --enable=rhel-7-server-rhceph-1.3-calamari-rpms --enable=rhel-7-server-rhceph-1.3-installer-rpms --enable=rhel-7-server-rhceph-1.3-tools-rpms --enable=rhel-7-server-rhceph-1.3-mon-rpms --enable=rhel-7-server-rhceph-1.3-osd-rpms


setenforce 0
systemctl disable firewalld
systemctl stop firewalld


sudo yum install yum-utils -y

sudo yum-config-manager --disable epel


yum install ntp -y

systemctl enable ntpd
systemctl restart ntpd

ntpq -p


useradd ceph
passwd ceph

password



cat << EOF >/etc/sudoers.d/ceph
ceph ALL = (root) NOPASSWD:ALL
Defaults:ceph !requiretty
EOF

sudo chmod 0440 /etc/sudoers.d/ceph

su - ceph


ssh-keygen

ssh-copy-id ceph@cephmon01
ssh-copy-id ceph@cephmon02
ssh-copy-id ceph@radosgw01
ssh-copy-id ceph@radosgw02
ssh-copy-id ceph@cephosd01
ssh-copy-id ceph@cephosd02
ssh-copy-id ceph@cephosd03
ssh-copy-id ceph@cephosd04
ssh-copy-id ceph@cephosd05



vi ~/.ssh/config

Host radosgw01
   Hostname radosgw01
   User ceph
Host radosgw02
   Hostname radosgw02
   User ceph
Host cephmon01
   Hostname cephmon01
   User ceph
Host cephmon02
   Hostname cephmon02
   User ceph
Host cephosd01
   Hostname cephosd01
   User ceph
Host cephosd02
   Hostname cephosd02
   User ceph
Host cephosd03
   Hostname cephosd03
   User ceph
Host cephosd04
   Hostname cephosd04
   User ceph
Host cephosd05
   Hostname cephosd05
   User ceph


chmod 600 ~/.ssh/config

sudo yum install ceph-deploy calamari-server calamari-clients -y


mkdir ~/ceph-config
cd ~/ceph-config

ceph-deploy new cephmon01 cephmon02



[global]
fsid = 4652d0b0-5c80-4fb0-9dea-5bb049c5a735
mon_initial_members = cephmon01
mon_host = 10.1.0.236,10.1.0.237
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

public_network = 10.1.0.0/24
cluster_network = 192.168.200.0/24

journal_size = 10000
osd_pool_default_size = 3
osd_pool_default_min_size = 2

osd_crush_chooseleaf_type = 1
osd_crush_update_on_start = false
max_open_files = 131072



ceph-deploy install --mon cephmon01 cephmon02
ceph-deploy install --rgw radosgw01 radosgw02
ceph-deploy install --osd cephosd01 cephosd02 cephosd03 cephosd04 cephosd05



10.1.0.232 radosgw01
10.1.0.233 radosgw02

10.1.0.236 cephmon01
10.1.0.237 cephmon02

10.1.0.240 cephosd01
10.1.0.241 cephosd02
10.1.0.242 cephosd03
10.1.0.243 cephosd04
10.1.0.244 cephosd05



ceph-deploy mon create-initial
ceph-deploy calamari connect --master cephadmin.time-gate.com cephmon01 cephmon02
ceph-deploy calamari connect --master cephadmin.time-gate.com cephosd01 cephosd02 cephosd03

 


ceph-deploy disk zap cephosd01:vdb cephosd02:vdb cephosd03:vdb cephosd04:vdb cephosd05:vdb

ceph-deploy osd prepare cephosd01:vdb cephosd02:vdb cephosd03:vdb cephosd04:vdb cephosd05:vdb

ceph-deploy osd activate cephosd01:vdb1:vdb2 cephosd02:vdb1:vdb2 cephosd03:vdb1:vdb2 cephosd04:vdb1:vdb2 cephosd05:vdb1:vdb2

ceph-deploy install --cli cephadmin
ceph-deploy admin cephadmin

sudo chmod +r /etc/ceph/ceph.client.admin.keyring

ceph quorum_status --format json-pretty




[ceph@cephadmin ceph-config]$ ceph-deploy rgw create radosgw01 radosgw02


http://10.1.0.232:7480/


####  OSD Purge
ceph-deploy purge cephosd01 cephosd02 cephosd03 cephosd04 cephosd05
ceph-deploy purgedata cephosd01 cephosd02 cephosd03 cephosd04 cephosd05

#### FULL Purge
ceph-deploy purge cephosd01 cephosd02 cephosd03 cephosd04 cephosd05 radosgw01 radosgw02 cephmon01 cephmon02 cephadmin
ceph-deploy purgedata cephosd01 cephosd02 cephosd03 cephosd04 cephosd05 radosgw01 radosgw02 cephmon01 cephmon02 cephadmin





ceph osd lspools

ceph osd pool delete rbd rbd --yes-i-really-really-mean-it

ceph osd pool create rbd 128 128



ceph osd pool set rbd pg_num 128
ceph osd pool set rbd pgp_num 128

##### osd 제거 추가/


1. 5개 각 osd node에서 실행.
/etc/init.d/ceph stop osd


2. admin node 에서

ceph osd pool delete rbd rbd --yes-i-really-really-mean-it


3. 5개 각 osd node에서 실행.
/etc/init.d/ceph start osd

4. 재설치시 인증에러 발생하면 key 삭제 후 다시 activate

[ceph@cephadmin ceph-config]$ ceph auth del osd.0
updated
[ceph@cephadmin ceph-config]$ ceph auth del osd.1
updated
[ceph@cephadmin ceph-config]$ ceph auth del osd.2
updated
[ceph@cephadmin ceph-config]$ ceph auth del osd.3
updated
[ceph@cephadmin ceph-config]$ ceph auth del osd.4
updated




ceph osd down, rm, out 후 다시 activate

sudo ceph osd crush remove osd.4
sudo ceph auth del osd.4
sudo ceph osd rm 4

ceph pg repair 0.2c

ceph osd out 1
ceph osd down 1
ceph osd rm 1



버그 해결 문서
최초 설치 후  rbd pool 제거 하고 다시 설치.
https://access.redhat.com/webassets/avalon/d/Red_Hat_Ceph_Storage-1.3.1-Release_Notes-en-US/Red_Hat_Ceph_Storage-1.3.1-Release_Notes-en-US.pdf


### bucket 구성


ceph osd crush add-bucket tg-center datacenter
ceph osd crush add-bucket mr01 room
ceph osd crush add-bucket row01 row
ceph osd crush add-bucket rack01 rack

ceph osd crush add-bucket cephosd01 host
ceph osd crush add-bucket cephosd02 host
ceph osd crush add-bucket cephosd03 host
ceph osd crush add-bucket cephosd04 host
ceph osd crush add-bucket cephosd05 host

ceph osd crush move tg-center root=default
ceph osd crush move mr01 datacenter=tg-center
ceph osd crush move row01 room=mr01
ceph osd crush move rack01 row=row01

ceph osd crush move cephosd01 rack=rack01
ceph osd crush move cephosd02 rack=rack01
ceph osd crush move cephosd03 rack=rack01
ceph osd crush move cephosd04 rack=rack01
ceph osd crush move cephosd05 rack=rack01


### 모니터 노드에서. crushmap 수정 방법


ceph osd getcrushmap -o crushmap
crushtool -d crushmap -o crushmap.txt

#Devices
device 0 osd.0
device 1 osd.1
device 2 osd.2
device 3 osd.3
device 4 osd.4

를 추가


crushmap 수정 후

crushtool -c crushmap.txt -o crushmap

ceph osd setcrushmap -i crushmap


ceph osd crush set osd.0 1.0 root=default datacenter=tg-center room=mr01 row=row01 rack=rack01 host=cephosd01
ceph osd crush set osd.1 1.0 root=default datacenter=tg-center room=mr01 row=row01 rack=rack01 host=cephosd02
ceph osd crush set osd.2 1.0 root=default datacenter=tg-center room=mr01 row=row01 rack=rack01 host=cephosd03
ceph osd crush set osd.3 1.0 root=default datacenter=tg-center room=mr01 row=row01 rack=rack01 host=cephosd04
ceph osd crush set osd.4 1.0 root=default datacenter=tg-center room=mr01 row=row01 rack=rack01 host=cephosd05

작성자: 시간: 댓글 없음:

2016년 9월 26일 월요일

Redhat7 infiniband bonding + bridge

[root@openstack01 network-scripts]# cat ifcfg-ib0
CONNECTED_MODE=yes
TYPE=InfiniBand
BOOTPROTO=no
#DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
NAME=ib0
UUID=af984bff-db23-4c53-b234-2d52b6029fed
DEVICE=ib0
ONBOOT=yes
HWADDR=80:00:02:0A:FE:80:00:00:00:00:00:00:00:21:28:00:01:CF:56:7F
MTU=65520
PEERDNS=yes
PEERROUTES=yes
IPV6_PEERDNS=yes
IPV6_PEERROUTES=yes
MASTER=bond1
SLAVE=yes
USERCTL=no
NM_CONTROLLED=no


[root@openstack01 network-scripts]# cat ifcfg-ib1
CONNECTED_MODE=yes
TYPE=InfiniBand
BOOTPROTO=none
#DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
NAME=ib1
UUID=0a441295-38bb-4407-9f6a-e36a9cc854ef
DEVICE=ib1
ONBOOT=yes
HWADDR=80:00:02:0B:FE:80:00:00:00:00:00:00:00:21:28:00:01:CF:56:80
MTU=65520
PEERDNS=yes
PEERROUTES=yes
IPV6_PEERDNS=yes
IPV6_PEERROUTES=yes
MASTER=bond1
SLAVE=yes
USERCTL=no
NM_CONTROLLED=no
[root@openstack01 network-scripts]#

[root@openstack01 network-scripts]# cat ifcfg-bond1 
NAME=bond1
DEVICE=bond1
BONDING_MASTER=yes
TYPE=Bond
IPADDR=192.168.10.4
NETMASK=255.255.255.0
ONBOOT=yes
BOOTPROTO=none
BONDING_OPTS="mode=active-backup miimon=100"
NM_CONTROLLED=no
[root@openstack01 network-scripts]# 

[root@openstack01 network-scripts]# cat ifcfg-virbr1
DEVICE="virbr1"
TYPE=BRIDGE
ONBOOT=yes
BOOTPROTO=none
IPADDR="192.168.10.4"
PREFIX=24
[root@openstack01 network-scripts]# 




작성자: 시간: 댓글 없음:

2016년 9월 22일 목요일

Pine64 ubuntu 16.04.1 xenial Ceph Storage Installation Hands on

Pine64 ubuntu 16.04.1 xenial
Ceph Storage Installation Hands on







목차



1.CEPH storage 권장 OS

http://docs.ceph.com/docs/jewel/start/os-recommendations/
http://ceph.com/releases/v10-2-0-jewel-released/

DISTRO COMPATIBILITY

Starting with Infernalis, we have dropped support for many older distributions so that we can move to a newer compiler toolchain (e.g., C++11). Although it is still possible to build Ceph on older distributions by installing backported development tools, we are not building and publishing release packages for ceph.com.
We now build packages for the following distributions and architectures:
  • x86_64:
      - CentOS 7.x. We have dropped support for CentOS 6 (and other RHEL 6 derivatives, like Scientific Linux 6).
      - Debian Jessie 8.x. Debian Wheezy 7.x’s g++ has incomplete support for C++11 (and no systemd).
      - Ubuntu Xenial 16.04 and Trusty 14.04. Ubuntu Precise 12.04 is no longer supported.
      - Fedora 22 or later.
  • aarch64 / arm64:
      - Ubuntu Xenial 16.04.




OS 확인 : ubuntu xenial (16.04.1 LTS)
ubuntu@admin:~$ lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 16.04.1 LTS
Release: 16.04
Codename: xenial




2.Pine64 ceph storage 구성품

5 x pine64 for storage node
1 x Raspberry Pi 3 for admin / monitor
1 x Netgear GS108E ethernet switch
5 x 16G MicroSD for OS
5 x 32G MicroSD for Storage
5 x USB3.0 MicroSD reader
1 x PC power supply 5V 30A







3.테스트 환경 구성도


Node Name
ceph01
ceph02
ceph03
ceph04
ceph05
admin
IP Address
10.0.1.21
10.0.1.22
10.0.1.23
10.0.1.24
10.0.1.25
10.0.1.26
Functionality
Ceph OSD Node
Ceph OSD node
Ceph OSD node
Ceph OSD node
monitor
Ceph OSD Node
monitor
Admin




4.Hostname 변경

작업대상 : ceph01 ceph02 ceph03 ceph04 ceph05 admin
ubuntu@localhost:~$ sudo hostnamectl set-hostname ceph01

5.불필요한 패키지 삭제

작업대상 : ceph01 ceph02 ceph03 ceph04 ceph05 admin
sudo apt-get remove --purge libreoffice* -y
sudo apt-get remove --purge gimp* -y
sudo apt-get remove --purge firefox* -y
sudo apt-get clean
sudo apt-get autoremove

6.필요 패키지 설치

작업대상 : ceph01 ceph02 ceph03 ceph04 ceph05 admin
~$ apt-get install btrfs-tools -y

7.시스템 부트를 Text mode 로 변경 (run level 3)

작업대상 : ceph01 ceph02 ceph03 ceph04 ceph05
ubuntu@ceph05:~$ sudo systemctl set-default multi-user.target

8./etc/hosts 등록

작업대상 : ceph01 ceph02 ceph03 ceph04 ceph05 admin
ubuntu@admin:~$ cat /etc/hosts
127.0.0.1 localhost
127.0.1.1 kevin-desktop


# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters


10.0.1.21 ceph01
10.0.1.22 ceph02
10.0.1.23 ceph03
10.0.1.24 ceph04
10.0.1.25 ceph05
10.0.1.26 admin

9.ceph User 생성

작업대상 : ceph01 ceph02 ceph03 ceph04 ceph05 admin
ubuntu@admin:~$ sudo useradd -b /var/lib -m ceph -s /bin/bash
ubuntu@admin:~$ sudo passwd ceph
Enter new UNIX password:
Retype new UNIX password:
passwd: password updated successfully
Password : welcome1 로 일괄 적용

10.ceph user를 sudoers 에 등록 후 로그인

작업대상 : ceph01 ceph02 ceph03 ceph04 ceph05 admin
ubuntu@admin:~$ echo "ceph ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/ceph
ubuntu@admin:~$ sudo chmod 0440 /etc/sudoers.d/ceph



11.각 노드에 ceph user access 를 위한 ssh key 전송

작업대상 : admin
ssh-keygen 생성시 passphase 는 입력하지 않고 enter 를 입력.
ubuntu@admin:~$ su - ceph
Password:

ceph@admin:~$ ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/var/lib/ceph/.ssh/id_rsa):
Created directory '/var/lib/ceph/.ssh'.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /var/lib/ceph/.ssh/id_rsa.
Your public key has been saved in /var/lib/ceph/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:4KIDqKg9y3gBODyjl6n3JG0ed8UG7O/djmE8kGZjOVM ceph@admin
The key's randomart image is:
+---[RSA 2048]----+
| |
| . |
|o . o E |
|=+ . o o + |
|+ooo. . S # |
|+.=o . B * |
|oo+.= . . . = |
|o++* o . . o = |
|oo++o . o.o |
+----[SHA256]-----+


ubuntu@admin:~$ ssh-copy-id ceph@ceph01
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/home/ubuntu/.ssh/id_rsa.pub"
The authenticity of host 'ceph01 (10.0.1.21)' can't be established.
ECDSA key fingerprint is SHA256:10+26IDg1MawZjIS6y8iDnzb/3majyh+C1mVyznGJ68.
Are you sure you want to continue connecting (yes/no)? yes
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys
ceph@ceph01's password:

Now try logging into the machine, with: "ssh 'ceph@ceph01'"
and check to make sure that only the key(s) you wanted were added.

ceph@admin:~$ ssh-copy-id ceph@ceph02
...
ceph@admin:~$ ssh-copy-id ceph@ceph03
...
ceph@admin:~$ ssh-copy-id ceph@ceph04
...
ceph@admin:~$ ssh-copy-id ceph@ceph05
...

12.ceph repository 등록

작업대상 : admin
ceph@admin:~$ wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' | sudo apt-key add -
OK
ceph@admin:~$ echo deb https://download.ceph.com/debian-jewel/ xenial $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
deb https://download.ceph.com/debian-jewel/ xenial xenial main

13.apt update

작업대상 : admin
ceph@admin:~$ sudo apt-get update -y && sudo apt-get upgrade -y

14.ceph-deploy 설치

작업대상 : admin
ceph@admin:~$ sudo apt-get install ceph-deploy -y
Reading package lists... Done
Building dependency tree
Reading state information... Done
The following NEW packages will be installed:
ceph-deploy
0 upgraded, 1 newly installed, 0 to remove and 1 not upgraded.
Need to get 0 B/96.3 kB of archives.
After this operation, 617 kB of additional disk space will be used.
Selecting previously unselected package ceph-deploy.
(Reading database ... 168830 files and directories currently installed.)
Preparing to unpack .../ceph-deploy_1.5.34_all.deb ...
Unpacking ceph-deploy (1.5.34) ...
Setting up ceph-deploy (1.5.34) ...

15.Monitor node 생성

작업대상 : admin
ceph@admin:~$ ceph-deploy new ceph04 ceph05
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.35): /usr/bin/ceph-deploy new ceph04 ceph05
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7654f378>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] ssh_copykey : True
[ceph_deploy.cli][INFO ] mon : ['ceph04', 'ceph05']
[ceph_deploy.cli][INFO ] func : <function new at 0x76533d70>
[ceph_deploy.cli][INFO ] public_network : None
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] cluster_network : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] fsid : None
[ceph_deploy.new][DEBUG ] Creating new cluster named ceph
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[ceph04][DEBUG ] connected to host: admin
[ceph04][INFO ] Running command: ssh -CT -o BatchMode=yes ceph04
[ceph04][DEBUG ] connection detected need for sudo
[ceph04][DEBUG ] connected to host: ceph04
[ceph04][DEBUG ] detect platform information from remote host
[ceph04][DEBUG ] detect machine type
[ceph04][DEBUG ] find the location of an executable
[ceph04][INFO ] Running command: sudo /bin/ip link show
[ceph04][INFO ] Running command: sudo /bin/ip addr show
[ceph04][DEBUG ] IP addresses found: [u'10.0.1.24']
[ceph_deploy.new][DEBUG ] Resolving host ceph04
[ceph_deploy.new][DEBUG ] Monitor ceph04 at 10.0.1.24
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[ceph05][DEBUG ] connected to host: admin
[ceph05][INFO ] Running command: ssh -CT -o BatchMode=yes ceph05
[ceph05][DEBUG ] connection detected need for sudo
[ceph05][DEBUG ] connected to host: ceph05
[ceph05][DEBUG ] detect platform information from remote host
[ceph05][DEBUG ] detect machine type
[ceph05][DEBUG ] find the location of an executable
[ceph05][INFO ] Running command: sudo /bin/ip link show
[ceph05][INFO ] Running command: sudo /bin/ip addr show
[ceph05][DEBUG ] IP addresses found: [u'10.0.1.25']
[ceph_deploy.new][DEBUG ] Resolving host ceph05
[ceph_deploy.new][DEBUG ] Monitor ceph05 at 10.0.1.25
[ceph_deploy.new][DEBUG ] Monitor initial members are ['ceph04', 'ceph05']
[ceph_deploy.new][DEBUG ] Monitor addrs are ['10.0.1.24', '10.0.1.25']
[ceph_deploy.new][DEBUG ] Creating a random mon key...
[ceph_deploy.new][DEBUG ] Writing monitor keyring to ceph.mon.keyring...
[ceph_deploy.new][DEBUG ] Writing initial config to ceph.conf...

16.ceph.conf 수정

작업대상 : admin
ceph@admin:~$ vi ceph.conf
[global]
fsid = 7bf83b5c-18f3-496a-b1b3-f54316162a68
mon_initial_members = ceph04, ceph05
mon_host = 10.0.1.24,10.0.1.25
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
osd pool default size = 2
public_network = 10.0.0.0/23
ceph@admin:~$

17.ceph install

Install 시 admin 을 제외한 다른 노드에서 ceph 계정으로 로그인 되어 있는지 확인 후 로그아웃 한다.
Ceph 계정으로 로그인 되어 있는 상태에서 install 시 에러가 발행 할 수 있다.
작업대상 : admin
ceph@admin:~$ ceph-deploy install ceph01 ceph02 ceph03 ceph04 ceph05 admin
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.34): /usr/bin/ceph-deploy install ceph01 ceph02 ceph03 ceph04 ceph05 admin
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] testing : None
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x765786e8>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] dev_commit : None
[ceph_deploy.cli][INFO ] install_mds : False
[ceph_deploy.cli][INFO ] stable : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] adjust_repos : True
[ceph_deploy.cli][INFO ] func : <function install at 0x765baa30>
[ceph_deploy.cli][INFO ] install_all : False
[ceph_deploy.cli][INFO ] repo : False
[ceph_deploy.cli][INFO ] host : ['ceph01', 'ceph02', 'ceph03', 'ceph04', 'ceph05', 'admin']
[ceph_deploy.cli][INFO ] install_rgw : False
[ceph_deploy.cli][INFO ] install_tests : False
[ceph_deploy.cli][INFO ] repo_url : None
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] install_osd : False
[ceph_deploy.cli][INFO ] version_kind : stable
[ceph_deploy.cli][INFO ] install_common : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] dev : master
[ceph_deploy.cli][INFO ] local_mirror : None
[ceph_deploy.cli][INFO ] release : None
[ceph_deploy.cli][INFO ] install_mon : False
[ceph_deploy.cli][INFO ] gpg_url : None

…
…



ceph-deploy install 에러 발생 시 install/uninstall 방법
install :
각 node 에서 root로 로그인 한 후 다음 명령으로 인스톨,

1. ~$ sudo wget -q -O- 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc' | sudo apt-key add -

2. ~$ echo deb https://download.ceph.com/debian-jewel/ xenial $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list

3. ~$ sudo env ubuntu_FRONTEND=noninteractive ubuntu_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install -o pkg::Options::=--force-confnew ceph ceph-mds radosgw

uninstall :
각 노드에서 강제 삭제시 아래와 같이 수행.
~$ sudo env ubuntu_FRONTEND=noninteractive ubuntu_PRIORITY=critical apt-get --assume-yes -q -f --force-yes remove --purge ceph ceph-mds ceph-common ceph-fs-common radosgw

18.Monitor node initial

작업대상 : admin
ceph@admin:~$ ceph-deploy mon create-initial
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO  ] Invoked (1.5.35): /usr/bin/ceph-deploy mon create-initial
[ceph_deploy.cli][INFO  ] ceph-deploy options:
[ceph_deploy.cli][INFO  ]  username                      : None
[ceph_deploy.cli][INFO  ]  verbose                       : False
[ceph_deploy.cli][INFO  ]  overwrite_conf                : False
[ceph_deploy.cli][INFO  ]  subcommand                    : create-initial
[ceph_deploy.cli][INFO  ]  quiet                         : False
[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf instance at 0x76452918>
[ceph_deploy.cli][INFO  ]  cluster                       : ceph
[ceph_deploy.cli][INFO  ]  func                          : <function mon at 0x7654c8b0>
[ceph_deploy.cli][INFO  ]  ceph_conf                     : None
[ceph_deploy.cli][INFO  ]  keyrings                      : None
[ceph_deploy.cli][INFO  ]  default_release               : False
[ceph_deploy.mon][DEBUG ] Deploying mon, cluster ceph hosts ceph04 ceph05
[ceph_deploy.mon][DEBUG ] detecting platform for host ceph04 ...
[ceph04][DEBUG ] connection detected need for sudo
[ceph04][DEBUG ] connected to host: ceph04 
[ceph04][DEBUG ] detect platform information from remote host
[ceph04][DEBUG ] detect machine type
[ceph04][DEBUG ] find the location of an executable
…
[ceph05][INFO  ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --admin-daemon=/var/run/ceph/ceph-mon.ceph05.asok mon_status
[ceph05][INFO  ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph05/keyring auth get-or-create client.admin osd allow * mds allow * mon allow *
[ceph05][INFO  ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph05/keyring auth get-or-create client.bootstrap-mds mon allow profile bootstrap-mds
[ceph05][INFO  ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph05/keyring auth get-or-create client.bootstrap-osd mon allow profile bootstrap-osd
[ceph05][INFO  ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph05/keyring auth get-or-create client.bootstrap-rgw mon allow profile bootstrap-rgw
[ceph_deploy.gatherkeys][INFO  ] Storing ceph.client.admin.keyring
[ceph_deploy.gatherkeys][INFO  ] Storing ceph.bootstrap-mds.keyring
[ceph_deploy.gatherkeys][INFO  ] keyring 'ceph.mon.keyring' already exists
[ceph_deploy.gatherkeys][INFO  ] Storing ceph.bootstrap-osd.keyring
[ceph_deploy.gatherkeys][INFO  ] Storing ceph.bootstrap-rgw.keyring
[ceph_deploy.gatherkeys][INFO  ] Destroy temp directory /tmp/tmpSKaOCe


19.OSD 설정

작업대상 : admin
ceph@admin:~$ ceph-deploy osd create ceph01:sda1 ceph02:sda1 ceph03:sda1 ceph04:sda1 ceph05:sda1 --fs-type btrfs
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.34): /usr/bin/ceph-deploy osd create ceph01:sda1 ceph02:sda1 ceph03:sda1 ceph04:sda1 --fs-type btrfs
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] disk : [('ceph01', '/dev/sda1', None), ('ceph02', '/dev/sda1', None), ('ceph03', '/dev/sda1', None), ('ceph04', '/dev/sda1', None)]
[ceph_deploy.cli][INFO ] dmcrypt : False
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] bluestore : None
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : create
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x764ccda0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] fs_type : btrfs
[ceph_deploy.cli][INFO ] func : <function osd at 0x764b11b0>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] zap_disk : False
[ceph_deploy.osd][DEBUG ] Preparingsudo apt-get remove compiz* cluster ceph disks ceph01:/dev/sda1: ceph02:/dev/sda1: ceph03:/dev/sda1: ceph04:/dev/sda1:
[ceph01][DEBUG ] connection detected need for sudo
…



20.OSD PREPARE

작업대상 : admin
ceph@admin:~$ ceph-deploy osd prepare ceph01:sda1 ceph02:sda1 ceph03:sda1 ceph04:sda1 ceph05:sda1 --fs-type btrfs
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.34): /usr/bin/ceph-deploy osd prepare ceph01:sda1 ceph02:sda1 ceph03:sda1 ceph04:sda1 --fs-type btrfs
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] disk : [('ceph01', '/dev/sda1', None), ('ceph02', '/dev/sda1', None), ('ceph03', '/dev/sda1', None), ('ceph04', '/dev/sda1', None)]
[ceph_deploy.cli][INFO ] dmcrypt : False
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] bluestore : None
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : prepare
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7648bda0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] fs_type : btrfs
[ceph_deploy.cli][INFO ] func : <function osd at 0x764701b0>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] zap_disk : False
[ceph_deploy.osd][DEBUG ] Preparing cluster ceph disks ceph01:/dev/sda1: ceph02:/dev/sda1: ceph03:/dev/sda1: ceph04:/dev/sda1:
[ceph01][DEBUG ] connection detected need for sudo
…



21.OSD ACTIVATE

작업대상 : admin
ceph@admin:~$ ceph-deploy osd activate ceph01:sda1 ceph02:sda1 ceph03:sda1 ceph04:sda1 ceph04:sda1
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.34): /usr/bin/ceph-deploy osd activate ceph01:sda1 ceph02:sda1 ceph03:sda1 ceph04:sda1
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : activate
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x76519da0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] func : <function osd at 0x764fe1b0>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] disk : [('ceph01', '/dev/sda1', None), ('ceph02', '/dev/sda1', None), ('ceph03', '/dev/sda1', None), ('ceph04', '/dev/sda1', None), ('ceph05', '/dev/sda1', None)]
[ceph_deploy.osd][DEBUG ] Activating cluster ceph disks ceph01:/dev/sda1: ceph02:/dev/sda1: ceph03:/dev/sda1: ceph04:/dev/sda1:
[ceph01][DEBUG ] connection detected need for sudo
[ceph01][DEBUG ] connected to host: ceph01
…

22.CEPH 상태 점검

작업대상 : admin
ceph@admin:~$ ceph quorum_status --format json-pretty


{
"election_epoch": 4,
"quorum": [
0,
1
],
"quorum_names": [
"ceph04",
"ceph05"
],
"quorum_leader_name": "ceph04",
"monmap": {
"epoch": 1,
"fsid": "7bf83b5c-18f3-496a-b1b3-f54316162a68",
"modified": "2016-09-22 19:22:24.810838",
"created": "2016-09-22 19:22:24.810838",
"mons": [
{
"rank": 0,
"name": "ceph04",
"addr": "10.0.1.24:6789\/0"
},
{
"rank": 1,
"name": "ceph05",
"addr": "10.0.1.25:6789\/0"
}
]
}
}



ceph@admin:~$ sudo chmod +r /etc/ceph/ceph.client.admin.keyring
ceph@admin:~$





ceph@admin:~$ ceph health
HEALTH_WARN too few PGs per OSD (25 < min 30)


ceph@admin:~$ ceph -s
cluster 7bf83b5c-18f3-496a-b1b3-f54316162a68
health HEALTH_OK
monmap e1: 2 mons at {ceph04=10.0.1.24:6789/0,ceph05=10.0.1.25:6789/0}
election epoch 4, quorum 0,1 ceph04,ceph05
osdmap e21: 4 osds: 4 up, 4 in
flags sortbitwise
pgmap v54: 64 pgs, 1 pools, 0 bytes data, 0 objects
20489 MB used, 100548 MB / 119 GB avail
64 active+clean

ceph@admin:~$ ceph osd pool set rbd pg_num 160
set pool 0 pg_num to 160


ceph@admin:~$ ceph osd pool set rbd pgp_num 160
set pool 0 pgp_num to 160


ceph@admin:~$ ceph -s
cluster 7bf83b5c-18f3-496a-b1b3-f54316162a68
health HEALTH_OK
monmap e1: 2 mons at {ceph04=10.0.1.24:6789/0,ceph05=10.0.1.25:6789/0}
election epoch 4, quorum 0,1 ceph04,ceph05
osdmap e30: 5 osds: 5 up, 5 in
flags sortbitwise
pgmap v90: 160 pgs, 1 pools, 0 bytes data, 0 objects
25617 MB used, 122 GB / 149 GB avail
160 active+clean


ceph@admin:~$ ceph mon stat
e1: 2 mons at {ceph04=10.0.1.24:6789/0,ceph05=10.0.1.25:6789/0}, election epoch 4, quorum 0,1 ceph04,ceph05
ceph@admin:~$ ceph mds stat
e1:
ceph@admin:~$ ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1 0.14549 root default
-2 0.02910 host ceph01
0 0.02910 osd.0 up 1.00000 1.00000
-3 0.02910 host ceph02
1 0.02910 osd.1 up 1.00000 1.00000
-4 0.02910 host ceph03
2 0.02910 osd.2 up 1.00000 1.00000
-5 0.02910 host ceph04
3 0.02910 osd.3 up 1.00000 1.00000
-6 0.02910 host ceph05
4 0.02910 osd.4 up 1.00000 1.00000

PG 설정에 대한 문서 참조.
http://docs.ceph.com/docs/master/rados/operations/placement-groups/


ceph@admin:~$ ceph osd dump
epoch 30
fsid 7bf83b5c-18f3-496a-b1b3-f54316162a68
created 2016-09-22 19:22:55.416036
modified 2016-09-22 19:39:39.805911
flags sortbitwise
pool 0 'rbd' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 160 pgp_num 160 last_change 24 flags hashpspool stripe_width 0
max_osd 5
osd.0 up in weight 1 up_from 4 up_thru 29 down_at 0 last_clean_interval [0,0) 10.0.1.21:6800/10656 10.0.1.21:6801/10656 10.0.1.21:6802/10656 10.0.1.21:6803/10656 exists,up 8a03e5e1-5447-4612-a0c5-4fd7ef1d2b18
osd.1 up in weight 1 up_from 8 up_thru 29 down_at 0 last_clean_interval [0,0) 10.0.1.22:6800/8422 10.0.1.22:6801/8422 10.0.1.22:6802/8422 10.0.1.22:6803/8422 exists,up efcc116c-4ec1-4dd9-b7b3-72ef3a03a7e7
osd.2 up in weight 1 up_from 14 up_thru 29 down_at 0 last_clean_interval [0,0) 10.0.1.23:6800/8436 10.0.1.23:6801/8436 10.0.1.23:6802/8436 10.0.1.23:6803/8436 exists,up adf35a8b-2785-4ef3-a25e-1b3f21c15ea1
osd.3 up in weight 1 up_from 19 up_thru 29 down_at 0 last_clean_interval [0,0) 10.0.1.24:6800/9878 10.0.1.24:6801/9878 10.0.1.24:6802/9878 10.0.1.24:6803/9878 exists,up fac90ac0-b1b4-4653-96d6-961dedae9b0d
osd.4 up in weight 1 up_from 28 up_thru 29 down_at 0 last_clean_interval [0,0) 10.0.1.25:6800/11122 10.0.1.25:6801/11122 10.0.1.25:6802/11122 10.0.1.25:6803/11122 exists,up ca220f2d-7b54-4e37-b227-e03cad244614

23.RADOS TEST

작업대상 : admin 또는 linux1


ceph@admin:~$ sudo dd if=/dev/zero of=/testfile.txt bs=1024 count=10000
10000+0 records in
10000+0 records out
10240000 bytes (10 MB, 9.8 MiB) copied, 0.21831 s, 46.9 MB/s


ceph@admin:~$ ceph df
GLOBAL:
SIZE AVAIL RAW USED %RAW USED
119G 100546M 20492M 16.78
POOLS:
NAME ID USED %USED MAX AVAIL OBJECTS
rbd 0 0 0 50272M 0



ceph@admin:~$ rados mkpool rdata
successfully created pool rdata


ceph@admin:~$ ceph df
GLOBAL:
SIZE AVAIL RAW USED %RAW USED
119G 100546M 20492M 16.78
POOLS:
NAME ID USED %USED MAX AVAIL OBJECTS
rbd 0 0 0 50272M 0
rdata 1 0 0 50272M 0


ceph@admin:~$ rados put test-object-1 /testfile.txt --pool=rdata


ceph@admin:~$ rados -p rdata ls
test-object-1


ceph@admin:~$ ceph osd map rdata test-object-1
osdmap e33 pool 'rdata' (1) object 'test-object-1' -> pg 1.74dc35e2 (1.2) -> up ([3,1], p3) acting ([3,1], p3)


ceph@admin:~$ rados rm test-object-1 --pool=rdata


ceph@admin:~$ rados -p rdata ls

24.BLOCK DEVICE TEST

작업대상 : admin
ceph@admin:~$ cat /etc/hosts
127.0.0.1 localhost
127.0.1.1 kevin-desktop


# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters


10.0.1.26 admin
10.0.1.21 ceph01
10.0.1.22 ceph02
10.0.1.23 ceph03
10.0.1.24 ceph04
10.0.1.25 ceph05
10.0.0.195 kevdev
10.0.1.150 linux1 # VirtualBox node




ceph@admin:~$ ceph-deploy install root@linux1
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.34): /usr/bin/ceph-deploy install root@linux1
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] verbose : False
...
[ceph_deploy.cli][INFO ] local_mirror : None
[ceph_deploy.cli][INFO ] release : None
[ceph_deploy.cli][INFO ] install_mon : False
[ceph_deploy.cli][INFO ] gpg_url : None
[ceph_deploy.install][DEBUG ] Installing stable version jewel on cluster ceph hosts root@linux1
[ceph_deploy.install][DEBUG ] Detecting platform for host root@linux1 ...
root@linux1's password:
root@linux1's password:
[root@linux1][DEBUG ] connected to host: root@linux1
…
[root@linux1][DEBUG ] Package 1:ceph-10.2.2-0.el7.x86_64 already installed and latest version
[root@linux1][DEBUG ] Package 1:ceph-radosgw-10.2.2-0.el7.x86_64 already installed and latest version
[root@linux1][DEBUG ] Nothing to dosudo apt-get remove compiz*
[root@linux1][INFO ] Running command: ceph --version
[root@linux1][DEBUG ] ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374)
ceph@admin:~$

ceph@admin:~$ ceph-deploy admin root@linux1
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.34): /usr/bin/ceph-deploy admin root@linux1
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x764a78c8>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] client : ['root@linux1']
[ceph_deploy.cli][INFO ] func : <function admin at 0x76559070>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to root@linux1
root@linux1's password:
root@linux1's password:
[root@linux1][DEBUG ] connected to host: root@linux1
[root@linux1][DEBUG ] detect platform information from remote host
[root@linux1][DEBUG ] detect machine type
[root@linux1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
ceph@admin:~$


작업대상 : linux1
[ceph@linux1 ~]$ rbd create rbd_data --size 4096
2016-09-06 06:32:59.338104 7fdebb003d80 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin: (2) No such file or directory
2016-09-06 06:32:59.338140 7fdebb003d80 -1 monclient(hunting): ERROR: missing keyring, cannot use cephx for authentication
2016-09-06 06:32:59.338142 7fdebb003d80 0 librados: client.admin initialization error (2) No such file or directory
rbd: couldn't connect to the cluster!


[ceph@linux1 ~]$ sudo chmod +r /etc/ceph/ceph.client.admin.keyring


[ceph@linux1 ~]$ rbd create rbd_data --size 4096
rbd 를 생성할때 keyring 권한문제로 접근이 안되어 에러가 발생. chmod 실행시켜 read 권한 추가.


[ceph@linux1 ceph]$ sudo rbd map rbd_data
rbd: sysfs write failed
RBD image feature set mismatch. You can disable features unsupported by the kernel with "rbd feature disable".
In some cases useful info is found in syslog - try "dmesg | tail" or so.
rbd: map failed: (6) No such device or address


[ceph@linux1 ceph]$ rbd feature disable rbd_data deep-flatten fast-diff object-map exclusive-lock


[ceph@linux1 ceph]$ sudo rbd map rbd_data
/dev/rbd0
rbd map 실행중 feature mismatch 가 발생. disable 작업 후 다시 map.


[ceph@linux1 ceph]$ sudo mkfs.ext4 -m0 /dev/rbd/rbd/rbd_data
mke2fs 1.42.9 (28-Dec-2013)
Discarding device blocks: done
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
Stride=1024 blocks, Stripe width=1024 blocks
262144 inodes, 1048576 blocks
0 blocks (0.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=1073741824
32 block groups
32768 blocks per group, 32768 fragments per group
8192 inodes per group
Superblock backups stored on blocks:
32768, 98304, 163840, 229376, 294912, 819200, 884736


Allocating group tables: done
Writing inode tables: done
Creating journal (32768 blocks): done
Writing superblocks and filesystem accounting information:
done


[ceph@linux1 ceph]$ sudo mount /dev/rbd/rbd/rbd_data /mnt


[ceph@linux1 ceph]$ cd /mnt


[ceph@linux1 mnt]$ ls -al
total 24
drwxr-xr-x. 3 root root 4096 Sep 6 06:40 .
dr-xr-xr-x. 17 root root 4096 Sep 5 21:25 ..
drwx------. 2 root root 16384 Sep 6 06:40 lost+found


[root@linux1 mnt]# sudo dd if=/dev/zero of=testfile.txt bs=1024 count=10000
10000+0 records in
10000+0 records out
10240000 bytes (10 MB) copied, 0.0359571 s, 285 MB/s


[root@linux1 mnt]# ls -al
합계 10024
drwxr-xr-x. 3 root root 4096 9월 6 13:58 .
dr-xr-xr-x. 18 root root 4096 9월 6 06:48 ..
drwx------. 2 root root 16384 9월 6 13:56 lost+found
-rw-r--r--. 1 root root 10240000 9월 6 13:58 testfile.txt
[root@linux1 mnt]#


[root@linux1 mnt]# ceph df
GLOBAL:
SIZE AVAIL RAW USED %RAW USED
149G 122G 25900M 16.97
POOLS:
NAME ID USED %USED MAX AVAIL OBJECTS
rbd 0 148M 0.19 62653M 48
[root@linux1 mnt]#


25.각 node 별 프로세스 확인

ceph@admin:~$ ps -ef|grep ceph | grep -v grep
root 1942 1628 0 9월22 pts/0 00:00:00 su - ceph
ceph 1951 1942 0 9월22 pts/0 00:00:01 -su
ceph 26485 1951 0 15:07 pts/0 00:00:00 ps -ef


ubuntu@ceph01:~$ ps -ef|grep ceph | grep -v grep
avahi 541 1 0 Sep22 ? 00:00:46 avahi-daemon: running [ceph01.local]
ceph 10656 1 0 Sep22 ? 00:05:34 /usr/bin/ceph-osd -f --cluster ceph --id 0 --setuser ceph --setgroup ceph
ubuntu@ceph01:~$


ubuntu@ceph02:~$ ps -ef|grep ceph | grep -v grep
avahi 524 1 0 Sep22 ? 00:00:47 avahi-daemon: running [ceph02.local]
ceph 8422 1 0 Sep22 ? 00:05:34 /usr/bin/ceph-osd -f --cluster ceph --id 1 --setuser ceph --setgroup ceph
ubuntu@ceph02:~$

ubuntu@ceph03:~$ ps -ef|grep ceph | grep -v grep
avahi 524 1 0 Sep22 ? 00:00:47 avahi-daemon: running [ceph03.local]
ceph 8436 1 0 Sep22 ? 00:05:40 /usr/bin/ceph-osd -f --cluster ceph --id 2 --setuser ceph --setgroup ceph
ubuntu@ceph03:~$


ubuntu@ceph04:~$ ps -ef|grep ceph | grep -v grep
ceph 7276 1 0 Sep22 ? 00:03:26 /usr/bin/ceph-mon -f --cluster ceph --id ceph04 --setuser ceph --setgroup ceph
ceph 9878 1 0 Sep22 ? 00:05:52 /usr/bin/ceph-osd -f --cluster ceph --id 3 --setuser ceph --setgroup ceph
ubuntu@ceph04:~$


ubuntu@ceph05:~$ ps -ef|grep ceph | grep -v grep
ceph 7479 1 0 Sep22 ? 00:00:54 /usr/bin/ceph-mon -f --cluster ceph --id ceph05 --setuser ceph --setgroup ceph
ceph 11122 1 0 Sep22 ? 00:05:40 /usr/bin/ceph-osd -f --cluster ceph --id 4 --setuser ceph --setgroup ceph
ubuntu@ceph05:~$


26.에러 확인.

ubuntu@ceph01:~$ sudo journalctl -f


ubuntu@ceph02:~$ sudo journalctl -f


ubuntu@ceph03:~$ sudo journalctl -f


ubuntu@ceph04:~$ sudo journalctl -f


ubuntu@ceph05:~$ sudo journalctl -f


ubuntu@admin:~$ sudo journalctl -f

27.Refresh all data and configuration

각 노드의 모든 설정을 초기화 하고 재 설치 할 때 수행.
작업대상 : admin


ceph@admin:~$ ceph-deploy disk zap ceph01:sda1 ceph02:sda1 ceph03:sda1 ceph04:sda1 ceph05:sda1
ceph@admin:~$ ceph-deploy forgetkeys
ceph@admin:~$ ceph-deploy purge ceph01 ceph02 ceph03 ceph04 ceph05 admin
ceph@admin:~$ ceph-deploy purgedata ceph01 ceph02 ceph03 ceph04 ceph05 admin




작업대상 : ceph01 ceph02 ceph03 ceph04 ceph05 admin
ceph@admin:~$ sudo su
# apt-get autoremove -y
# apt-get autoclean -y
# rm -fr /etc/apt/source-list.d/ceph.list
# deluser --remove-home ceph
# rm -fr /var/lib/ceph
# rm -fr /var/local/osd*
# reboot


작성자: 시간: 댓글 없음:
피드 구독하기: 글 (Atom)