240402수_TIL

아침의 방황 about 추천 알고리즘..

 

import tensorflow as tf

import os
import tensorflow_datasets as tfds

### Basic libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import pickle
import warnings
warnings.filterwarnings(action='ignore')

# Data Preprocessing
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import LabelEncoder

# Model Training
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
import tensorflow as tf

## Import necessary modules for collaborative filtering
from tensorflow.keras.layers import Input, Embedding, Dot, Flatten, Dense
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from wordcloud import WordCloud
from collections import defaultdict
from collections import Counter

## Import necessary modules for content-based filtering
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel

# Calculating the average score
avg_score = np.mean(df['rating'])
print('Average Score:', avg_score)

# Scaling our "rating" column
# Create a MinMaxScaler object
scaler = MinMaxScaler(feature_range=(0, 1))

# Scale the 'score' column between 0 and 1
df['scaled_score'] = scaler.fit_transform(df[['rating']])

# Encoding categorical data

## Encoding user IDs
user_encoder = LabelEncoder()
df["user_encoded"] = user_encoder.fit_transform(df["user_id"])
num_users = len(user_encoder.classes_)

## Encoding anime IDs
anime_encoder = LabelEncoder()
df["anime_encoded"] = anime_encoder.fit_transform(df["anime_id"])
num_animes = len(anime_encoder.classes_)

# Printing dataset information
print("Number of unique users: {}, Number of unique anime: {}".format(num_users, num_animes))
print("Minimum rating: {}, Maximum rating: {}".format(min(df['rating']), max(df['rating'])))

# Shuffle the dataset
df = shuffle(df, random_state=100)

# Create feature matrix X and target variable y
X = df[['user_encoded', 'anime_encoded']].values
y = df["scaled_score"].values

# Printing dataset information
print("Shape of X:", X.shape)
print("Shape of y:", y.shape)

test_set_size = 10000  # Number of samples to include in the test set

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_set_size, random_state=73)

print("Number of samples in the training set:", len(y_train))
print("Number of samples in the test set:", len(y_test))

# Prepare input data for model training and evaluation
X_train_array = [X_train[:, 0], X_train[:, 1]]
X_test_array = [X_test[:, 0], X_test[:, 1]]​

# Setting TPU_INIT flag
USE_TPU = True

# Checking if TPU is initialized
if USE_TPU:
    # If tpu is connected then start creating TPUStrategy
    tpu_resolver = tf.distribute.cluster_resolver.TPUClusterResolver.connect()
    tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu_resolver)
else:
    !nvidia-smi
    
# Print the TensorFlow version
tf_version = tf.__version__
print(f"TensorFlow version: {tf_version}")

ValueError: Please provide a TPU Name to connect to.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.distribute.cluster_resolver.tpu.tpu_cluster_resolver import is_running_in_gce  # pylint: disable=unused-import
from tensorflow.python.distribute.cluster_resolver.tpu.tpu_cluster_resolver import TPUClusterResolver
from tensorflow.python.util.tf_export import tf_export

tf_export('distribute.cluster_resolver.TPUClusterResolver')(TPUClusterResolver)

ValueError: Mime type rendering requires nbformat>=4.2.0 but it is not installed

resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='')
tf.config.experimental_connect_to_cluster(resolver)
# This is the TPU initialization code that has to be at the beginning.
tf.tpu.experimental.initialize_tpu_system(resolver)
print("All devices: ", tf.config.list_logical_devices('TPU'))

 

TPU 사용  |  TensorFlow Core

TensorFlow 문제 해결 - TPU  |  Google Cloud

 

tensorflow-1/tensorflow/python/distribute/cluster_resolver/BUILD at master · sourcecode369/tensorflow-1

 

cannot import name 'Adam' from 'keras.optimizers'

tpu 몰루 이다. 

---

 

 

[Markdown] 마크다운 문법 정리

---

 

다시 전처리

 

[에러] AttributeError: 'DataFrame' object has no attribute 'split'

 

 

TYPE

movie인 경우, aired(방영일에 개봉일자)  premiered(는 전체 결측값)

 

ONA Original Net Animation  오로지 인터넷을 통해서만 공개되는 애니메이션

OVA Original Video Animation 오리지널 비디오 애니메이션 - TV 방송이나 영화 상영없이 오로지 소매로 파는 애니메이션을 뜻한다. OVA들은 주로 DVD나 블루레이로 발매

 

- ONA/OVA에서 scored by가 unknown인 컬럼을 지워보도록. 

 

장르

특정 장르 포함 여부 원핫 인코딩

# one-hot encode the genres
#total_genres = list(set(", ".join(my_data["Genres"].unique()).split(", ")))
#print(total_genres)
total_genres = ['Adventure', 'Drama', 'Gourmet', 'Romance', 'Sports', 'Action', 'Suspense', 'Fantasy', 'Horror', 'Hentai', 'Supernatural', 'Avant Garde', 'Erotica', 'Mystery', 'Ecchi', 'Sci-Fi', 'Girls Love', 'UNKNOWN', 'Award Winning', 'Slice of Life', 'Boys Love', 'Comedy']
df_anime[["genre "+gg for gg in total_genres]] = 0
#print(my_data.columns)

def split_genre(row):
    gs = row["Genres"].split(", ")
    for gg in gs:
        row["genre "+gg] = 1
    return row

df_anime = df_anime.apply(split_genre, axis=1)

 

duration

min으로 통일

df_anime['hours'] = df_anime['Duration'].str.extract(r'(\d+) hr').fillna(0).astype(int)
df_anime['minutes'] = df_anime['Duration'].str.extract(r'(\d+) min').fillna(0).astype(int)

 

일단 내일까지는 큰 틀에서 파악할 수 있는 지점들을 더 꼼꼼히 찾아보도록 .! 

ㅇㅈㅈ..!