/dev/sdb1 /data1 ext4 defaults,noatime 0

mount -o remount /data1

127.0.0.1 localhost.localdomain localhost
192.168.1.1 cluster-01.example.com cluster-01
192.168.1.2 cluster-02.example.com cluster-02
192.168.1.3 cluster-03.example.com cluster-03

[root@grocery-1 ~]# cat /etc/hostname
ccycloud-1.grocery.root.hwx.site

[root@grocery-1 ~]# hostnamectl set-hostname ccycloud-1.grocery.root.hwx.site

[root@ccycloud-1 ~]# cat /etc/hosts
127.0.0.1  localhost
::1  localhost ip6-localhost ip6-loopback
fe00::0  ip6-localnet
ff00::0  ip6-mcastprefix
ff02::1  ip6-allnodes
ff02::2  ip6-allrouters
172.27.188.66 ccycloud-1.grocery.root.hwx.site ccycloud-1
[root@ccycloud-1 ~]#

[root@ccycloud-1 ~]# ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:JJhXYOx4GpwZ4ofeaoUHPSVlRFov3ywhrCEwE1u9Cfg root@ccycloud.grocery.root.hwx.site
The key's randomart image is:
+---[RSA 2048]----+
|=o.. =O..        |
|.*o +Ooo         |
|.o.*=@* +        |
|  E.%+o* +       |
| . =.=  S o      |
|  o =    .       |
|   +             |
|  o              |
| .               |
+----[SHA256]-----+
[root@ccycloud-1 ~]#

for i in {1..5}; do ssh-copy-id -i ~/.ssh/id_rsa.pub ccycloud-$i.grocery.root.hwx.site ; done

[root@ccycloud-1 ~]# for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site " hostname -A" ; done
ccycloud-1.grocery.root.hwx.site
ccycloud-2.grocery.root.hwx.site
ccycloud-3.grocery.root.hwx.site
ccycloud-4.grocery.root.hwx.site
ccycloud-5.grocery.root.hwx.site
[root@ccycloud-1 ~]#

yum -y install createrepo wget unzip

sudo yum -y install httpd

AddType application/x-gzip .gz .tgz .parcel

sed -i 's/AddType application\/x-gzip .gz .tgz/AddType application\/x-gzip .gz .tgz .parcel/' /etc/httpd/conf/httpd.conf

systemctl enable httpd
systemctl start httpd
systemctl status httpd

sudo mkdir /media/DVD1
sudo mount -o loop CentOS-7-x86_64-DVD-1611.iso /media/DVD1/

$ sudo vim /etc/yum.repos.d/local_os.repo
[local_iso]
name=CentOS-$releasever - Media
baseurl=file:///media/DVD1
gpgcheck=0
enabled=1
$ sudo yum repolist

[ec2-user@ip-172-31-2-159 ~]$ sudo mkdir /var/www/html/iso
[ec2-user@ip-172-31-2-159 ~]$ sudo scp -r /media/DVD1/* /var/www/html/iso/
[ec2-user@ip-172-31-2-159 ~]$ sudo vim /etc/yum.repos.d/os.repo
[osrepo]
name=os_repo
baseurl=http://grocery-1.vpc.cloudera.com/iso/
enabled=true
gpgcheck=false
[ec2-user@ip-172-31-2-159 ~]$ sudo yum repolist

for i in {2..5}; do scp /etc/yum.repos.d/os.repo grocery-$i.vpc.cloudera.com:/etc/yum.repos.d/os.repo; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site " yum -y update " ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site " setenforce 0  " ; done

#关闭selinux
for i in {1..6}; do ssh ccycloud-$i.grocery.root.hwx.site 'sed –i "7s/enforcing/disabled/g" /etc/selinux/config '  ;done

# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
#     enforcing - SELinux security policy is enforced.
#     permissive - SELinux prints warnings instead of enforcing.
#     disabled - No SELinux policy is loaded.
SELINUX=disabled
# SELINUXTYPE= can take one of three two values:
#     targeted - Targeted processes are protected,
#     minimum - Modification of targeted policy. Only selected processes are protected.
#     mls - Multi Level Security protection.
SELINUXTYPE=targeted

##关闭防火墙
for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'systemctl disable firewalld' ; done
for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'systemctl stop firewalld.service' ; done
for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'systemctl list-unit-files | grep firewalld' ; done
for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'systemctl status firewalld.service' ; done

##设置时区和时钟同步
for i in {1..6}; do ssh ccycloud-$i.grocery.root.hwx.site 'yum -y install chrony'  ;done
for i in {1..6}; do ssh ccycloud-$i.grocery.root.hwx.site 'timedatectl set-timezone Asia/Shanghai' ; done
for i in {1..6}; do ssh ccycloud-$i.grocery.root.hwx.site 'chkconfig --level 345 chronyd on' ; done

[root@grocery-1 ~]# vim /etc/chrony.conf
server ccycloud-1.grocery.root.hwx.site iburst
#allow 192.168.0.0/16
allow 172.27.0.0/16

[root@ccycloud-1 ~]# egrep -v "^#|^$" /etc/chrony.conf
[root@ccycloud-1 ~]# egrep -v "^#|^$" /etc/chrony.conf
server ccycloud-1.grocery.root.hwx.site iburst
driftfile /var/lib/chrony/drift
makestep 1.0 3
rtcsync
allow 172.27.0.0/16
keyfile /etc/chrony.keys
leapsectz right/UTC
logdir /var/log/chrony

systemctl enable chronyd
systemctl restart chronyd
# 查看时间同步状态
timedatectl status
# 开启网络时间同步
timedatectl set-ntp true

vi /etc/chrony.conf,修改两处：server和allow部分
# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
server ccycloud-1.grocery.root.hwx.site iburst
Allow NTP client access from local network.
allow ccycloud-1.grocery.root.hwx.site

systemctl enable chronyd

systemctl restart chronyd

查看时间同步状态
timedatectl status
开启网络时间同步
timedatectl set-ntp true

for i in {1..6}; do ssh ccycloud-$i.grocery.root.hwx.site ' systemctl enable chronyd'  ;done

for i in {1..6}; do ssh ccycloud-$i.grocery.root.hwx.site ' systemctl restart chronyd'  ;done

for i in {1..6}; do ssh ccycloud-$i.grocery.root.hwx.site ' timedatectl status'  ;done

for i in {1..6}; do ssh ccycloud-$i.grocery.root.hwx.site ' timedatectl set-ntp true'  ;done

chronyc sources -v

chronyc sourcestats -v

timedatectl set-local-rtc 1

timedatectl set-ntp yes

chronyc tracking

[root@ccycloud-1 ~]# #设置vm.swappiness

[root@ccycloud-1 ~]# for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'echo "vm.swappiness = 1" >> /etc/sysctl.conf' ; done

[root@ccycloud-1 ~]# for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'sysctl vm.swappiness=1' ; done

#设置透明大页面

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.d/rc.local' ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> /etc/rc.d/rc.local' ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'chmod +x /etc/rc.d/rc.local' ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'cat /etc/rc.d/rc.local' ; done

#关闭iptables

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'systemctl stop iptables' ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'chkconfig iptables off' ; done

#设置limits

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'echo "hdfs - nofile 32768" >> /etc/security/limits.conf' ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'echo "mapred - nofile 32768" >> /etc/security/limits.conf' ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'echo "hbase - nofile 32768" >> /etc/security/limits.conf' ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'echo "hdfs - noproc 32768" >> /etc/security/limits.conf' ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'echo "mapred - noproc 32768" >> /etc/security/limits.conf' ; done

for i in {1..5}; do ssh ccycloud-$i.grocery.root.hwx.site 'echo "hbase - noproc 32768" >> /etc/security/limits.conf' ; done

## MariaDB 10.5

cat - >/etc/yum.repos.d/MariaDB.repo <<EOF

[mariadb]

name = MariaDB

baseurl = http://mirrors.ustc.edu.cn/mariadb/yum/10.5/centos7-amd64/

gpgkey=http://mirrors.ustc.edu.cn/mariadb/yum/RPM-GPG-KEY-MariaDB

gpgcheck=1

EOF

# 检查是否有mariaDB源

[root@ccycloud-1 ~]# yum repolist enabled | grep MariaDB

Failed to set locale, defaulting to C

mariadb                           MariaDB                                   95+4

[root@ccycloud-1 ~]#

yum install -y mariadb-server

cat  > /etc/my.cnf <<EOF

[mysqld]

datadir=/var/lib/mysql

socket=/var/lib/mysql/mysql.sock

transaction-isolation = READ-COMMITTED

Disabling symbolic-links is recommended to prevent assorted security risks;
to do so, uncomment this line:
symbolic-links = 0
Settings user and group are ignored when systemd is used.
If you need to run mysqld under a different user or group,
customize your systemd unit file for mariadb according to the
instructions in http://fedoraproject.org/wiki/Systemd
key_buffer = 16M

key_buffer_size = 32M

max_allowed_packet = 32M

thread_stack = 256K

thread_cache_size = 64

query_cache_limit = 8M

query_cache_size = 64M

query_cache_type = 1
max_connections = 550

#expire_logs_days = 10

#max_binlog_size = 100M
#log_bin should be on a disk with enough free space.

#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your

#system and chown the specified folder to the mysql user.

log_bin=/var/lib/mysql/mysql_binary_log
#In later versions of MariaDB, if you enable the binary log and do not set

#a server_id, MariaDB will not start. The server_id must be unique within

#the replicating group.

server_id=1
binlog_format = mixed
read_buffer_size = 2M

read_rnd_buffer_size = 16M

sort_buffer_size = 8M

join_buffer_size = 8M
InnoDB settings
innodb_file_per_table = 1

innodb_flush_log_at_trx_commit  = 2

innodb_log_buffer_size = 64M

innodb_buffer_pool_size = 4G

innodb_thread_concurrency = 8

innodb_flush_method = O_DIRECT

innodb_log_file_size = 512M
[mysqld_safe]

log-error=/var/log/mariadb/mariadb.log

pid-file=/var/run/mariadb/mariadb.pid

include all files from the config directory

!includedir /etc/my.cnf.d

EOF

bind-address=0.0.0.0

default-storage-engine=innodb

sql_mode=STRICT_ALL_TABLES

echo "--Enable and start MariaDB"

systemctl enable mariadb

systemctl start mariadb

sudo /usr/bin/mysql_secure_installation

[...]

Enter current password for root (enter for none):

OK, successfully used password, moving on...

[...]

Set root password? [Y/n] Y

New password:

Re-enter new password:

[...]

Remove anonymous users? [Y/n] Y

[...]

Disallow root login remotely? [Y/n] N

[...]

Remove test database and access to it [Y/n] Y

[...]

Reload privilege tables now? [Y/n] Y

[...]

All done!  If you've completed all of the above steps, your MariaDB

installation should now be secure.

Thanks for using MariaDB!

CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'cloudera';

CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'cloudera';
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'cloudera';
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

GRANT ALL ON metastore.* TO 'hive'@'%' IDENTIFIED BY 'cloudera';
CREATE DATABASE efm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

GRANT ALL ON efm.* TO 'efm'@'%' IDENTIFIED BY 'cloudera';
CREATE DATABASE nifireg DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

GRANT ALL ON nifireg.* TO 'nifireg'@'%' IDENTIFIED BY 'cloudera';
CREATE DATABASE registry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

GRANT ALL ON registry.* TO 'registry'@'%' IDENTIFIED BY 'cloudera';
CREATE DATABASE streamsmsgmgr DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

GRANT ALL ON streamsmsgmgr.* TO 'streamsmsgmgr'@'%' IDENTIFIED BY 'cloudera';
CREATE DATABASE ranger DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

GRANT ALL ON ranger.* TO 'ranger'@'%' IDENTIFIED BY 'cloudera';

wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.48.tar.gz

tar zxvf mysql-connector-java-5.1.48.tar.gz

for i in {1..6}; do ssh ccycloud-i.grocery.root.hwx.site &#39;mkdir -p /usr/share/java/&#39; ; done</code></pre></div></div><p>#将jdbc拷贝到所有节点的对应目录</p><div class="rno-markdown-code"><div class="rno-markdown-code-toolbar"><div class="rno-markdown-code-toolbar-info"><div class="rno-markdown-code-toolbar-item is-type"><span class="is-m-hidden">代码语言：</span>javascript</div></div><div class="rno-markdown-code-toolbar-opt"><div class="rno-markdown-code-toolbar-copy"><i class="icon-copy"></i><span class="is-m-hidden">复制</span></div></div></div><div class="developer-code-block"><pre class="prism-token token line-numbers language-javascript"><code class="language-javascript" style="margin-left:0">for i in {1..6}; do scp ~/mysql-connector-java-5.1.48/mysql-connector-java-5.1.48-bin.jar  ccycloud-i.grocery.root.hwx.site:/usr/share/java/mysql-connector-java.jar; done

https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm

rpm -ivh pgdg-redhat-repo-latest.noarch.rpm

ls -lrt /etc/yum.repos.d/

yum list postgresql*

sudo yum -y install postgresql10-server

sudo yum  -y install python-pip

for i in {1..4}; do ssh grocery-$i.vpc.cloudera.com " sudo yum  -y install python-pip" ; done

pip install --upgrade pip

for i in {1..4}; do ssh grocery-$i.vpc.cloudera.com " pip install --upgrade pip " ; done

sudo pip install psycopg2==2.7.5 --ignore-installed

for i in {1..4}; do ssh grocery-$i.vpc.cloudera.com " sudo pip install psycopg2==2.7.5 --ignore-installed " ; done

echo 'LC_ALL="en_US.UTF-8"' >> /etc/locale.conf

/usr/pgsql-10/bin/postgresql-10-setup initdb

host all all 127.0.0.1/32 md5

host all all 127.0.0.1/32 ident

sudo systemctl enable postgresql-10

sudo systemctl restart postgresql-10

sudo -u postgres psql

CREATE ROLE <user> LOGIN PASSWORD '<password>';

CREATE DATABASE <database> OWNER <user> ENCODING 'UTF8';

CREATE ROLE scm LOGIN PASSWORD 'cloudera';

CREATE DATABASE scm OWNER scm ENCODING 'UTF8';

CREATE ROLE amon LOGIN PASSWORD 'cloudera';

CREATE DATABASE amon OWNER amon ENCODING 'UTF8';
CREATE ROLE rman LOGIN PASSWORD 'cloudera';

CREATE DATABASE rman OWNER rman ENCODING 'UTF8';
CREATE ROLE hue LOGIN PASSWORD 'cloudera';

CREATE DATABASE hue OWNER hue ENCODING 'UTF8';
CREATE ROLE hive LOGIN PASSWORD 'cloudera';

CREATE DATABASE metastore OWNER hive ENCODING 'UTF8';
CREATE ROLE oozie LOGIN PASSWORD 'cloudera';

CREATE DATABASE oozie OWNER oozie ENCODING 'UTF8';
CREATE ROLE das LOGIN PASSWORD 'cloudera';

CREATE DATABASE das OWNER das ENCODING 'UTF8';
CREATE ROLE rangeradmin LOGIN PASSWORD '!Beijing28';

CREATE DATABASE ranger OWNER rangeradmin ENCODING 'UTF8';

GRANT ALL PRIVILEGES ON DATABASE ranger TO rangeradmin;

yum install postgresql-jdbc*

cp /usr/share/java/postgresql-jdbc.jar /usr/share/java/postgresql-connector-java.jar

ls /usr/share/java/postgresql-connector-java.jar

chmod 644 /usr/share/java/postgresql-connector-java.jar

for i in {1..6}; do ssh grocery-$i.vpc.cloudera.com " sudo yum -y install postgresql-jdbc* " ; done

for i in {1..6}; do ssh grocery-$i.vpc.cloudera.com " cp /usr/share/java/postgresql-jdbc.jar /usr/share/java/postgresql-connector-java.jar " ; done

for i in {1..6}; do ssh grocery-$i.vpc.cloudera.com " ls /usr/share/java/postgresql-connector-java.jar " ; done

for i in {1..6}; do ssh grocery-i.vpc.cloudera.com &#34; chmod 644 /usr/share/java/postgresql-connector-java.jar&#34; ; done</code></pre></div></div><figure class=""><div class="rno-markdown-img-url" style="text-align:center"><div class="rno-markdown-img-url-inner" style="width:100%"><div style="width:100%"><img src="https://cdn.static.attains.cn/app/developer-bbs/upload/1722964211550645055.jpg" /></div></div></div></figure><figure class=""><div class="rno-markdown-img-url" style="text-align:center"><div class="rno-markdown-img-url-inner" style="width:100%"><div style="width:100%"><img src="https://cdn.static.attains.cn/app/developer-bbs/upload/1722964211907108456.jpg" /></div></div></div></figure><h3 id="97sft" name="%E5%AE%89%E8%A3%85Mysql"><strong>安装Mysql</strong></h3><p>如果使用Mysql，则跳过Mariadb/PostgreSQL的部分。MySQL 数据库的安装和部署参考：https://docs.cloudera.com/cdp-private-cloud-base/latest/installation/topics/cm_ig_mysql.html  </p><h3 id="87cb6" name="%E5%AE%89%E8%A3%85JDK"><strong>安装JDK</strong></h3><p>我们这里使用操作系统自带的openjdk，提前安装。也可以只安装Cloudera Manager Server节点，然后通过向导安装openJDK。</p><div class="rno-markdown-code"><div class="rno-markdown-code-toolbar"><div class="rno-markdown-code-toolbar-info"><div class="rno-markdown-code-toolbar-item is-type"><span class="is-m-hidden">代码语言：</span>javascript</div></div><div class="rno-markdown-code-toolbar-opt"><div class="rno-markdown-code-toolbar-copy"><i class="icon-copy"></i><span class="is-m-hidden">复制</span></div></div></div><div class="developer-code-block"><pre class="prism-token token line-numbers language-javascript"><code class="language-javascript" style="margin-left:0">#安装openJDK1.8</code></pre></div></div><div class="rno-markdown-code"><div class="rno-markdown-code-toolbar"><div class="rno-markdown-code-toolbar-info"><div class="rno-markdown-code-toolbar-item is-type"><span class="is-m-hidden">代码语言：</span>javascript</div></div><div class="rno-markdown-code-toolbar-opt"><div class="rno-markdown-code-toolbar-copy"><i class="icon-copy"></i><span class="is-m-hidden">复制</span></div></div></div><div class="developer-code-block"><pre class="prism-token token line-numbers language-javascript"><code class="language-javascript" style="margin-left:0">for i in {1..5}; do ssh ccycloud-i.grocery.root.hwx.site 'yum -y install java-1.8.0-openjdk.x86_64 java-1.8.0-openjdk-headless.x86_64 java-1.8.0-openjdk-devel.x86_64' ; done

Cloudera Manager Repository:

https://username:password@archive.cloudera.com/p/cm7/7.7.1/redhat8/yum

Cloudera Manager Repository File:

https://username:password@archive.cloudera.com/p/cm7/7.7.1/redhat8/yum/cloudera-manager.repo

Cloudera Runtime 7.1.8 location:

https://[username]:[password]@archive.cloudera.com/p/cdh7/7.1.8/parcels

其中username和password使用上面的credentials来替换。例如：

mkdir -p /var/www/html/cm7/7.7.1/redhat8/yum/RPMS/x86_64/

mkdir -p /var/www/html/cdh7/7.1.8.0/parcels/

mkdir -p  /var/www/html/cm7/7.7.1/repo-as-tarball/

wget -nd -r  -l1 --no-parent https://username:password@archive.cloudera.com/p/cm7/7.7.1/redhat8/yum/RPMS/x86_64/ -P /var/www/html/cm7/7.7.1/redhat8/yum/RPMS/x86_64/

wget https://username:password@archive.cloudera.com/p/cm7/7.7.1/redhat8/yum/RPM-GPG-KEY-cloudera -P /var/www/html/cm7/7.7.1/redhat8/yum

wget https://username:password@archive.cloudera.com/p/cm7/7.7.1/redhat8/yum/cloudera-manager-trial.repo -P /var/www/html/cm7/7.7.1/redhat8/yum

wget https://username:password@archive.cloudera.com/p/cm7/7.7.1/redhat8/yum/cloudera-manager.repo -P /var/www/html/cm7/7.7.1/redhat8/yum

wget https://username:password@archive.cloudera.com/p/cm7/7.7.1/allkeys.asc -P /var/www/html/cm7/7.7.1

wget https://username:password@archive.cloudera.com/p/cdh7/7.1.8.0/parcels/CDH-7.1.8-1.cdh7.1.8.p0.30990532-el8.parcel.sha1  -P /var/www/html/cdh7/7.1.8.0/parcels/

wget https://username:password@archive.cloudera.com/p/cdh7/7.1.8.0/parcels/CDH-7.1.8-1.cdh7.1.8.p0.30990532-el8.parcel.sha256  -P /var/www/html/cdh7/7.1.8.0/parcels/

wget https://username:password@archive.cloudera.com/p/cdh7/7.1.8.0/parcels/CDH-7.1.8-1.cdh7.1.8.p0.30990532-el8.parcel  -P /var/www/html/cdh7/7.1.8.0/parcels/

wget https://username:password@archive.cloudera.com/p/cdh7/7.1.8.0/parcels/manifest.json  -P /var/www/html/cdh7/7.1.8.0/parcels/

[root@ccycloud-1 yum]# createrepo .
Directory walk started
Directory walk done - 5 packages
Temporary output repo path: ./.repodata/
Preparing sqlite DBs
Pool started (with 5 workers)
Pool finished
[root@ccycloud-1 yum]# ls
RPM-GPG-KEY-cloudera  RPMS  cloudera-manager-trial.repo  cloudera-manager.repo  repodata
[root@ccycloud-1 yum]#

[cloudera-manager]
name=Cloudera Manager 7.7.1
baseurl=http://ccycloud-1.grocery.root.hwx.site/cm7/7.7.1/redhat8/yum/
gpgkey=http://ccycloud-1.grocery.root.hwx.site/cm7/7.7.1/redhat8/yum/RPM-GPG-KEY-cloudera
gpgcheck=1
enabled=1
autorefresh=0
type=rpm-md

cp cloudera-manager.repo /etc/yum.repos.d/

yum repolist
Failed to set locale, defaulting to C.UTF-8
Failed loading plugin "product-id": No module named 'subscription_manager'
Failed loading plugin "upload-profile": No module named 'subscription_manager'
repo id                                           repo name
appstream-internal                                Red Hat Universal Base Image 8 (RPMs) - AppStream Internal
baseos-internal                                   Red Hat Universal Base Image 8 (RPMs) - BaseOS Internal
cloudera-manager                                  Cloudera Manager 7.7.1
codeready-builder-for-rhel-8-x86_64-rpms-internal Red Hat Universal Base Image 8 (RPMs) - Codeready-Builder
epel-internal                                     Epel Packages for Enterprise Linux 8 - x86_64
mariadb                                           MariaDB
supplymentry-internal                             Red Hat Universal Base Image 8 (RPMs) - Supplementary Internal
[root@ccycloud-1 parcels]#

sudo yum -y install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server

yum install gcc openssl-devel bzip2-devel libffi-devel zlib-devel -y

cd /opt
curl -O https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tgz
tar -zxvf Python-3.8.12.tgz

cd /opt/Python-3.8.12

./configure --enable-shared --prefix=[***CUSTOM-INSTALL-PATH***]

make

make install

cp --no-clobber ./libpython3.8.so* /lib64/

chmod 755 /lib64/libpython3.8.so*

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/

Key: PATH
Value: [***CUSTOM-INSTALL-PATH***]:/usr/local/sbin:/usr/local/bin:/usr/sbin:

yum install -y mysql-devel xmlsec1  xmlsec1-openssl

export PATH=/usr/local/bin:$PATH

pip3.8 install mysqlclient

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh [options] <databaseType> <databaseName> <databaseUser> <password>

[root@ccycloud-1 yum]# sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm cloudera
JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.262.b10-0.el7_8.x86_64
Verifying that we can write to /etc/cloudera-scm-server
Creating SCM configuration file in /etc/cloudera-scm-server
Executing:  /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.262.b10-0.el7_8.x86_64/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/java/postgresql-connector-java.jar:/opt/cloudera/cm/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db.
log4j:ERROR Could not find value for key log4j.appender.A
log4j:ERROR Could not instantiate appender named "A".
[2022-04-28 16:38:33,489] INFO     0[main] - com.cloudera.enterprise.dbutil.DbCommandExecutor.testDbConnection(DbCommandExecutor.java) - Successfully connected to database.
All done, your SCM database is configured correctly!
[root@ccycloud-1 yum]#

systemctl start cloudera-scm-server
systemctl status cloudera-scm-server

sudo tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log

INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server.

[root@ccycloud-1 yum]#  netstat -lnpt | grep 7180
tcp        0      0 0.0.0.0:7180            0.0.0.0:*               LISTEN      3476/java
[root@ccycloud-1 yum]#

mkdir -p /opt/cloudera/security/x509/ /opt/cloudera/security/jks/

sudo chown -R cloudera-scm:cloudera-scm /opt/cloudera/security/jks
sudo umask 0700
cd /opt/cloudera/security/jks

keytool -genkeypair -alias cmhost -keyalg RSA -keysize 2048 -dname "cn=cm01.example.com, ou=Department,
o=Company, l=City, st=State, c=US" -keypass password -keystore example.jks -storepass password
[root@ccycloud-1 jks]# keytool -genkeypair -alias cmhost -keyalg RSA -keysize 2048 -dname "cn=ccycloud-1.grocery.root.hwx.site, ou=ipausers, o=Cloudera, l=Shanghai, st=Shanghai, c=CN" -keypass cloudera -keystore example.jks -storepass cloudera
Warning:

The JKS keystore uses a proprietary format. It is recommended to migrate to PKCS12 which is an industry standard format using "keytool -importkeystore -srckeystore example.jks -destk

export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk

source /etc/profile

sudo cp JAVA_HOME/jre/lib/security/cacerts JAVA_HOME/jre/lib/security/jssecacerts

keytool -export -alias cmhost -keystore example.jks -rfc -file selfsigned.cer

[root@ccycloud-1 jks]# keytool -export -alias cmhost -keystore example.jks -rfc -file selfsigned.cer

Enter keystore password:

Certificate stored in file <selfsigned.cer>
Warning:

The JKS keystore uses a proprietary format. It is recommended to migrate to PKCS12 which is an industry standard format using "keytool -importkeystore -srckeystore example.jks -destkeystore example.jks -deststoretype pkcs12".

[root@ccycloud-1 jks]#

cp selfsigned.cer /opt/cloudera/security/x509/cmhost.pem

[root@ccycloud-1 jks]# for i in {2..6}; do ssh ccycloud-$i.grocery.root.hwx.site 'mkdir -p /opt/cloudera/security/jks'  ;done

[root@ccycloud-1 jks]# for i in {2..6}; do scp /opt/cloudera/security/jks/selfsigned.cer ccycloud-$i.grocery.root.hwx.site:/opt/cloudera/security/jks/selfsigned.cer ;done

cmhost.pem                                                                       100% 1349     3.1MB/s   00:00

cmhost.pem                                                                       100% 1349     1.5MB/s   00:00

cmhost.pem                                                                       100% 1349     3.1MB/s   00:00

cmhost.pem                                                                       100% 1349     2.0MB/s   00:00

cmhost.pem                                                                       100% 1349     1.9MB/s   00:00

[root@ccycloud-1 jks]#

$ keytool -import -alias cmhost -file /opt/cloudera/security/jks/selfsigned.cer

-keystore $JAVA_HOME/jre/lib/security/jssecacerts -storepass changeit

[root@ccycloud-1 jks]# keytool -import -alias cmhost -file /opt/cloudera/security/jks/selfsigned.cer

Enter keystore password:

Re-enter new password:

Owner: CN=ccycloud-1.grocery.root.hwx.site, OU=ipausers, O=Cloudera, L=Shanghai, ST=Shanghai, C=CN

Issuer: CN=ccycloud-1.grocery.root.hwx.site, OU=ipausers, O=Cloudera, L=Shanghai, ST=Shanghai, C=CN

Serial number: 5b9a917f

Valid from: Thu Apr 28 17:31:02 CST 2022 until: Wed Jul 27 17:31:02 CST 2022

Certificate fingerprints:

MD5:  61:C0:0F:9F:07:B5:FD:08:FA:8A:41:E7:C0:9F:5E:8F

SHA1: 15:C4:AF:62:88:4A:34:31:A3:60:8A:F1:13:FD:B2:6C:00:A2:D4:7F

SHA256: DC:54:FF:79:C3:88:11:D9:8D:BE:FD:EA:B9:EF:72:47:2E:10:5C:85:30:34:CC:18:DD:05:09:32:2C:AE:A6:71

Signature algorithm name: SHA256withRSA

Subject Public Key Algorithm: 2048-bit RSA key

Version: 3
Extensions:
#1: ObjectId: 2.5.29.14 Criticality=false

SubjectKeyIdentifier [

KeyIdentifier [

0000: A6 2A 18 0B C5 E2 85 28   BB 89 EF 80 1A BD 6D 6B  .*.....(......mk

0010: 38 6C 2E 32                                        8l.2

]

]
Trust this certificate? [no]:  yes

Certificate was added to keystore

[root@ccycloud-1 jks]#

mv /opt/cloudera/security/jks/example.jks /opt/cloudera/security/jks/cmhost-keystore.jks

rm /opt/cloudera/security/jks/selfsigned.cer

vi /usr/lib/systemd/system/ntpd.services

systemctl daemon-reload

systemctl start ntpd
systemctl status ntpd

[root@ccycloud-1 ~]# ntpdate time.windows.com
28 Apr 15:27:41 ntpdate[1962]: Can't adjust the time of day: Operation not permitted
[root@ccycloud-1 ~]#

When trying to install CDH via Cloudera Manager, you may sometimes encounter "heartbeat" errors similar to the following:
Installation failed. Failed to receive heartbeat from agent.
Ensure that the host's hostname is configured properly.
Ensure that port 7182 is accessible on the Cloudera Manager server (check firewall rules).
Ensure that ports 9000 and 9001 are free on the host being added.
Check agent logs in /var/log/cloudera-scm-agent/ on the host being added (some of the logs can be found in the installation details).

IP Address misconfiguration:
Use "ifconfig -a" to see a listing of your network interfaces, your main network interface is probably something like "eth0".  Assure that it has a real IP address, not the loopback address (127.0.0.1) assigned to it.  Run the "hostname -f" command to find out what hostname your local machine is using for itself, then run "nslookup <hostname>" against that hostname (or "dig <hostname>" for more options) to see what IP address it is resolving to. If DNS does not return an IP address for your host, then the configuration will be strictly controlled by /etc/hosts.  Look in that file to see what IP address you are assigning to your host.

Firewalls or iptables:  either disable them or assure they are allowing the correct ports to pass through.  Follow you company policies to decide which path if best for you.

$ sudo chkconfig iptables --list

iptables 0:off 1:off 2:off 3:off 4:off 5:off 6:off
$ sudo ufw disable

If "nslookup <hostname>" (where <hostname> is the name of your Cloudera Manager server) does not return the correct IP address, then you have a misconfigured DNS, contact your network admin.

When it's all said and done, all machines in your cluster need to be able to resolve each other's hostnames and IP addresses as well as connect to each other on the specific network ports mentioned in the error message.  Assure that /etc/hosts or DNS are configured properly so that your hosts can resolve each other and that each local machine is binding it's hadoop services to a real network IP instead of the loopback address.
If all that is correct and you still cannot connect, check to make sure firewalls or other services are not blocking the traffic.

service cloudera-scm-agent stop

service supervisord stop

systemctl stop cloudera-scm-server

yum -y remove cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server

rm -rf /opt/cloudera/* /var/lib/cloudera*   /var/log/cloudera* /etc/cloudera*

Error, CM server guid updated, expected 25cf17b3-391a-4368-848a-07118d6f11fb, received b3d4a47a-1476-4ea4-b236-24426b1b8540

rm /var /lib/cloudera-scm-agent/cm_guid  -f

systemctl restart cloudera-scm-agent

[root@esgyn001 ~]# cd /var/lib/cloudera-scm-agent/

cm_guid response.avro uuid

[root@esgyn001 cloudera-scm-agent]# rm -rf cm_guid

[root@esgyn001 cloudera-scm-agent]# service cloudera-scm-agent restart

Stopping cloudera-scm-agent: [ OK ]

Starting cloudera-scm-agent:

com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException: Specified key was too long; max key length is 767 bytes

SQLException : SQL state: 42000 com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException: Specified key was too long; max key length is 767 bytes ErrorCode: 1071

2019-11-24 01:12:06,672  [E] ranger_core_db_mysql.sql file import failed!

SET GLOBAL innodb_file_per_table = ON,

innodb_file_format = Barracuda,

innodb_large_prefix = ON;